dtc: update to upstream 227d6a3

- Report missing includes at the correct location.
- Add initial support for the -@ option emitting a symbol table.
- Add support for running tests with and without -@
- Add support for generating __fixups__ and __local_fixups__
- Attach the to-string transform to the node path.

8 files changed, 287 insertions, 63 deletions

diff --git a/usr.bin/dtc/checking.cc b/usr.bin/dtc/checking.cc
index 13f14399365b4..ffd6d24ef318b 100644
--- a/usr.bin/dtc/checking.cc
+++ b/usr.bin/dtc/checking.cc
@@ -97,16 +97,16 @@ namespace
 				}
 				if (found_size && found_address)
 				{
-					break;
+						break;
 				}
 			}
 			if (!found_address)
 			{
-				report_error("Missing #address-cells property");
+					report_error("Missing #address-cells property");
 			}
 			if (!found_size)
 			{
-				report_error("Missing #size-cells property");
+					report_error("Missing #size-cells property");
 			}
 			return found_address && found_size;
 		}
diff --git a/usr.bin/dtc/dtb.hh b/usr.bin/dtc/dtb.hh
index 4fece0d397471..fa93178a876bb 100644
--- a/usr.bin/dtc/dtb.hh
+++ b/usr.bin/dtc/dtb.hh
@@ -189,17 +189,17 @@ class binary_writer : public output_writer
 	 *  The binary format does not support labels, so this method
 	 * does nothing.
 	 */
-	virtual void write_label(const std::string &) {}
+	void write_label(const std::string &) override {}
 	/**
 	 * Comments are ignored by the binary writer.
 	 */
-	virtual void write_comment(const std::string&) {}
-	virtual void write_string(const std::string &name);
-	virtual void write_data(uint8_t v);
-	virtual void write_data(uint32_t v);
-	virtual void write_data(uint64_t v);
-	virtual void write_to_file(int fd);
-	virtual uint32_t size();
+	void write_comment(const std::string&)  override {}
+	void write_string(const std::string &name) override;
+	void write_data(uint8_t v) override;
+	void write_data(uint32_t v) override;
+	void write_data(uint64_t v) override;
+	void write_to_file(int fd) override;
+	uint32_t size() override;
 };
 /**
  * Assembly writer.  This class is responsible for writing the output in an
@@ -234,7 +234,7 @@ class asm_writer : public output_writer
 	/**
 	 * Write a string to the output.
 	 */
-	void write_string(const std::string &c);
+	void write_string(const std::string &c) override;
 	/**
 	 * Writes the string, starting on a new line.  
 	 */
@@ -246,13 +246,13 @@ class asm_writer : public output_writer
 	void write_byte(uint8_t b);
 	public:
 	asm_writer() : byte_count(0), bytes_written(0) {}
-	virtual void write_label(const std::string &name);
-	virtual void write_comment(const std::string &name);
-	virtual void write_data(uint8_t v);
-	virtual void write_data(uint32_t v);
-	virtual void write_data(uint64_t v);
-	virtual void write_to_file(int fd);
-	virtual uint32_t size();
+	void write_label(const std::string &name) override;
+	void write_comment(const std::string &name) override;
+	void write_data(uint8_t v) override;
+	void write_data(uint32_t v) override;
+	void write_data(uint64_t v) override;
+	void write_to_file(int fd) override;
+	uint32_t size() override;
 };
 
 /**
diff --git a/usr.bin/dtc/dtc.1 b/usr.bin/dtc/dtc.1
index c2fd68155f25d..2201691fb1fe1 100644
--- a/usr.bin/dtc/dtc.1
+++ b/usr.bin/dtc/dtc.1
@@ -38,7 +38,7 @@
 .Nd device tree compiler
 .Sh SYNOPSIS
 .Nm
-.Op Fl fhsv
+.Op Fl @fhsv
 .Op Fl b Ar boot_cpu_id
 .Op Fl d Ar dependency_file
 .Op Fl E Ar [no-]checker_name
@@ -84,6 +84,8 @@ Enable or disable a specified checker.
 The argument is the name of the checker.
 The full list of checkers is given in
 .Sx CHECKERS .
+.It Fl @
+Emit a __symbols__ node to allow plugins to be loaded.
 .It Fl f
 Force the tool to attempt to generate the output, even if the input had errors.
 .It Fl h
diff --git a/usr.bin/dtc/dtc.cc b/usr.bin/dtc/dtc.cc
index 3b1867c1f2735..5ba5cb9a3b7a9 100644
--- a/usr.bin/dtc/dtc.cc
+++ b/usr.bin/dtc/dtc.cc
@@ -54,7 +54,7 @@ int version_major = 0;
 /**
  * The current minor version of the tool.
  */
-int version_minor = 4;
+int version_minor = 5;
 /**
  * The current patch level of the tool.
  */
@@ -63,7 +63,7 @@ int version_patch = 0;
 static void usage(const string &argv0)
 {
 	fprintf(stderr, "Usage:\n"
-		"\t%s\t[-fhsv] [-b boot_cpu_id] [-d dependency_file]"
+		"\t%s\t[-fhsv@] [-b boot_cpu_id] [-d dependency_file]"
 			"[-E [no-]checker_name]\n"
 		"\t\t[-H phandle_format] [-I input_format]"
 			"[-O output_format]\n"
@@ -101,7 +101,7 @@ main(int argc, char **argv)
 	clock_t c0 = clock();
 	class device_tree tree;
 	fdt::checking::check_manager checks;
-	const char *options = "hqI:O:o:V:d:R:S:p:b:fi:svH:W:E:DP:";
+	const char *options = "@hqI:O:o:V:d:R:S:p:b:fi:svH:W:E:DP:";
 
 	// Don't forget to update the man page if any more options are added.
 	while ((ch = getopt(argc, argv, options)) != -1)
@@ -114,6 +114,9 @@ main(int argc, char **argv)
 		case 'v':
 			version(argv[0]);
 			return EXIT_SUCCESS;
+		case '@':
+			tree.write_symbols = true;
+			break;
 		case 'I':
 		{
 			string arg(optarg);
diff --git a/usr.bin/dtc/fdt.cc b/usr.bin/dtc/fdt.cc
index 16ebfd1fa3ddb..222d3e02b8a8b 100644
--- a/usr.bin/dtc/fdt.cc
+++ b/usr.bin/dtc/fdt.cc
@@ -169,6 +169,16 @@ property_value::resolve_type()
 	type = BINARY;
 }
 
+size_t
+property_value::size()
+{
+	if (!byte_data.empty())
+	{
+		return byte_data.size();
+	}
+	return string_data.size() + 1;
+}
+
 void
 property_value::write_as_string(FILE *file)
 {
@@ -286,7 +296,6 @@ property::parse_cells(text_input_buffer &input, int cell_size)
 				return;
 			}
 			input.next_token();
-			bool isPath = false;
 			string referenced;
 			if (!input.consume('{'))
 			{
@@ -296,7 +305,6 @@ property::parse_cells(text_input_buffer &input, int cell_size)
 			{
 				referenced = input.parse_to('}');
 				input.consume('}');
-				isPath = true;
 			}
 			if (referenced.empty())
 			{
@@ -655,6 +663,21 @@ property::write_dts(FILE *file, int indent)
 	fputs(";\n", file);
 }
 
+size_t
+property::offset_of_value(property_value &val)
+{
+	size_t off = 0;
+	for (auto &v : values)
+	{
+		if (&v == &val)
+		{
+			return off;
+		}
+		off += v.size();
+	}
+	return -1;
+}
+
 string
 node::parse_name(text_input_buffer &input, bool &is_property, const char *error)
 {
@@ -764,6 +787,21 @@ node::node(input_buffer &structs, input_buffer &strings) : valid(true)
 	return;
 }
 
+
+node::node(const string &n,
+           const std::vector<property_ptr> &p)
+	: name(n)
+{
+	props.insert(props.begin(), p.begin(), p.end());
+}
+
+node_ptr node::create_special_node(const string &name,
+                                   const std::vector<property_ptr> &props)
+{
+	node_ptr n(new node(name, props));
+	return n;
+}
+
 node::node(text_input_buffer &input,
            string &&n,
            std::unordered_set<string> &&l,
@@ -1123,7 +1161,6 @@ device_tree::collect_names_recursive(node_ptr &n, node_path &path)
 	{
 		collect_names_recursive(c, path);
 	}
-	path.pop_back();
 	// Now we collect the phandles and properties that reference
 	// other nodes.
 	for (auto &p : n->properties())
@@ -1132,7 +1169,7 @@ device_tree::collect_names_recursive(node_ptr &n, node_path &path)
 		{
 			if (v.is_phandle())
 			{
-				phandles.push_back(&v);
+				fixups.push_back({path, p, v});
 			}
 			if (v.is_cross_reference())
 			{
@@ -1154,6 +1191,7 @@ device_tree::collect_names_recursive(node_ptr &n, node_path &path)
 			}
 		}
 	}
+	path.pop_back();
 }
 
 void
@@ -1163,7 +1201,7 @@ device_tree::collect_names()
 	node_names.clear();
 	node_paths.clear();
 	cross_references.clear();
-	phandles.clear();
+	fixups.clear();
 	collect_names_recursive(root, p);
 }
 
@@ -1191,37 +1229,38 @@ device_tree::resolve_cross_references()
 			}
 		}
 	}
-	std::unordered_set<property_value*> phandle_set;
-	for (auto &i : phandles)
+	std::unordered_map<property_value*, fixup&> phandle_set;
+	for (auto &i : fixups)
 	{
-		phandle_set.insert(i);
+		phandle_set.insert({&i.val, i});
 	}
-	std::vector<property_value*> sorted_phandles;
+	std::vector<std::reference_wrapper<fixup>> sorted_phandles;
 	root->visit([&](node &n) {
 		for (auto &p : n.properties())
 		{
 			for (auto &v : *p)
 			{
-				if (phandle_set.count(&v))
+				auto i = phandle_set.find(&v);
+				if (i != phandle_set.end())
 				{
-					sorted_phandles.push_back(&v);
+					sorted_phandles.push_back(i->second);
 				}
 			}
 		}
 	});
-	assert(sorted_phandles.size() == phandles.size());
+	assert(sorted_phandles.size() == fixups.size());
 
 	uint32_t phandle = 1;
 	for (auto &i : sorted_phandles)
 	{
-		string target_name = i->string_data;
+		string target_name = i.get().val.string_data;
 		node *target = nullptr;
 		string possible;
 		// If the node name is a path, then look it up by following the path,
 		// otherwise jump directly to the named node.
 		if (target_name[0] == '/')
 		{
-			std::string path;
+			string path;
 			target = root.get();
 			std::istringstream ss(target_name);
 			string path_element;
@@ -1276,13 +1315,21 @@ device_tree::resolve_cross_references()
 		}
 		if (target == nullptr)
 		{
-			fprintf(stderr, "Failed to find node with label: %s\n", target_name.c_str());
-			if (possible != string())
+			if (is_plugin)
 			{
-				fprintf(stderr, "Possible intended match: %s\n", possible.c_str());
+				unresolved_fixups.push_back(i);
+				continue;
+			}
+			else
+			{
+				fprintf(stderr, "Failed to find node with label: %s\n", target_name.c_str());
+				if (possible != string())
+				{
+					fprintf(stderr, "Possible intended match: %s\n", possible.c_str());
+				}
+				valid = 0;
+				return;
 			}
-			valid = 0;
-			return;
 		}
 		// If there is an existing phandle, use it
 		property_ptr p = target->get_property("phandle");
@@ -1322,8 +1369,8 @@ device_tree::resolve_cross_references()
 				target->add_property(p);
 			}
 		}
-		p->begin()->push_to_buffer(i->byte_data);
-		assert(i->byte_data.size() == 4);
+		p->begin()->push_to_buffer(i.get().val.byte_data);
+		assert(i.get().val.byte_data.size() == 4);
 	}
 }
 
@@ -1340,6 +1387,10 @@ device_tree::parse_file(text_input_buffer &input,
 		read_header = true;
 	}
 	input.next_token();
+	if (input.consume("/plugin/;"))
+	{
+		is_plugin = true;
+	}
 	input.next_token();
 	if (!read_header)
 	{
@@ -1567,6 +1618,30 @@ device_tree::parse_dtb(const string &fn, FILE *)
 	valid = (root != 0);
 }
 
+string
+device_tree::node_path::to_string() const
+{
+	string path;
+	auto p = begin();
+	auto pe = end();
+	if ((p == pe) || (p+1 == pe))
+	{
+		return string("/");
+	}
+	// Skip the first name in the path.  It's always "", and implicitly /
+	for (++p ; p!=pe ; ++p)
+	{
+		path += '/';
+		path += p->first;
+		if (!(p->second.empty()))
+		{
+			path += '@';
+			path += p->second;
+		}
+	}
+	return path;
+}
+
 void
 device_tree::parse_dts(const string &fn, FILE *depfile)
 {
@@ -1631,6 +1706,85 @@ device_tree::parse_dts(const string &fn, FILE *depfile)
 	}
 	collect_names();
 	resolve_cross_references();
+	if (write_symbols)
+	{
+		std::vector<property_ptr> symbols;
+		// Create a symbol table.  Each label  in this device tree may be
+		// referenced by other plugins, so we create a __symbols__ node inside
+		// the root that contains mappings (properties) from label names to
+		// paths.
+		for (auto &s : node_paths)
+		{
+			property_value v;
+			v.string_data = s.second.to_string();
+			v.type = property_value::STRING;
+			string name = s.first;
+			auto prop = std::make_shared<property>(std::move(name));
+			prop->add_value(v);
+			symbols.push_back(prop);
+		}
+		root->add_child(node::create_special_node("__symbols__", symbols));
+		// If this is a plugin, then we also need to create two extra nodes.
+		// Internal phandles will need to be renumbered to avoid conflicts with
+		// already-loaded nodes and external references will need to be
+		// resolved.
+		if (is_plugin)
+		{
+			// Create the fixups entry.  This is of the form:
+			// {target} = {path}:{property name}:{offset}
+			auto create_fixup_entry = [&](fixup &i, string target)
+				{
+					string value = i.path.to_string();
+					value += ':';
+					value += i.prop->get_key();
+					value += ':';
+					value += std::to_string(i.prop->offset_of_value(i.val));
+					property_value v;
+					v.string_data = value;
+					v.type = property_value::STRING;
+					auto prop = std::make_shared<property>(std::move(target));
+					prop->add_value(v);
+					return prop;
+				};
+			// If we have any unresolved phandle references in this plugin,
+			// then we must update them to 0xdeadbeef and leave a property in
+			// the /__fixups__ node whose key is the label and whose value is
+			// as described above.
+			if (!unresolved_fixups.empty())
+			{
+				symbols.clear();
+				for (auto &i : unresolved_fixups)
+				{
+					auto &val = i.get().val;
+					symbols.push_back(create_fixup_entry(i, val.string_data));
+					val.byte_data.push_back(0xde);
+					val.byte_data.push_back(0xad);
+					val.byte_data.push_back(0xbe);
+					val.byte_data.push_back(0xef);
+					val.type = property_value::BINARY;
+				}
+				root->add_child(node::create_special_node("__fixups__", symbols));
+			}
+			symbols.clear();
+			// If we have any resolved phandle references in this plugin, then
+			// we must leave a property in the /__local_fixups__ node whose key
+			// is 'fixup' and whose value is as described above.
+			for (auto &i : fixups)
+			{
+				if (!i.val.is_phandle())
+				{
+					continue;
+				}
+				symbols.push_back(create_fixup_entry(i, "fixup"));
+			}
+			// We've iterated over all fixups, but only emit the
+			// __local_fixups__ if we found some that were resolved internally.
+			if (!symbols.empty())
+			{
+				root->add_child(node::create_special_node("__local_fixups__", symbols));
+			}
+		}
+	}
 }
 
 bool device_tree::parse_define(const char *def)
@@ -1653,7 +1807,7 @@ bool device_tree::parse_define(const char *def)
 	text_input_buffer in(std::move(raw),
 	                     std::unordered_set<string>(),
 	                     std::vector<string>(),
-	                     std::string(),
+	                     string(),
 	                     nullptr);
 	property_ptr p = property::parse(in, std::move(name_copy), string_set(), false);
 	if (p)
diff --git a/usr.bin/dtc/fdt.hh b/usr.bin/dtc/fdt.hh
index 993403b49910b..a6ba252685aa1 100644
--- a/usr.bin/dtc/fdt.hh
+++ b/usr.bin/dtc/fdt.hh
@@ -211,6 +211,10 @@ struct property_value
 	 * false otherwise.
 	 */
 	bool try_to_merge(property_value &other);
+	/**
+	 * Returns the size (in bytes) of this property value.
+	 */
+	size_t size();
 	private:
 	/**
 	 * Returns whether the value is of the specified type.  If the type of
@@ -380,6 +384,10 @@ class property
 	 * applicable way that it can determine.
 	 */
 	void write_dts(FILE *file, int indent);
+	/**
+	 * Returns the byte offset of the specified property value.
+	 */
+	size_t offset_of_value(property_value &val);
 };
 
 /**
@@ -479,6 +487,10 @@ class node
 	     std::string &&a,
 	     define_map*);
 	/**
+	 * Creates a special node with the specified name and properties.
+	 */
+	node(const std::string &n, const std::vector<property_ptr> &p);
+	/**
 	 * Comparison function for properties, used when sorting the properties
 	 * vector.  Orders the properties based on their names.
 	 */
@@ -579,6 +591,11 @@ class node
 	 */
 	static node_ptr parse_dtb(input_buffer &structs, input_buffer &strings);
 	/**
+	 * Construct a new special node from a name and set of properties.
+	 */
+	static node_ptr create_special_node(const std::string &name,
+			const std::vector<property_ptr> &props);
+	/**
 	 * Returns a property corresponding to the specified key, or 0 if this
 	 * node does not contain a property of that name.
 	 */
@@ -591,6 +608,13 @@ class node
 		props.push_back(p);
 	}
 	/**
+	 * Adds a new child to this node.
+	 */
+	inline void add_child(node_ptr &&n)
+	{
+		children.push_back(std::move(n));
+	}
+	/**
 	 * Merges a node into this one.  Any properties present in both are
 	 * overridden, any properties present in only one are preserved.
 	 */
@@ -626,7 +650,14 @@ class device_tree
 	 * Type used for node paths.  A node path is sequence of names and unit
 	 * addresses.
 	 */
-	typedef std::vector<std::pair<std::string,std::string> > node_path;
+	class node_path : public std::vector<std::pair<std::string,std::string>>
+	{
+		public:
+		/**
+		 * Converts this to a string representation.
+		 */
+		std::string to_string() const;
+	};
 	/**
 	 * Name that we should use for phandle nodes.
 	 */
@@ -681,11 +712,34 @@ class device_tree
 	 */
 	std::vector<property_value*> cross_references;
 	/**
+	 * The location of something requiring a fixup entry.
+	 */
+	struct fixup
+	{
+		/**
+		 * The path to the node.
+		 */
+		node_path path;
+		/**
+		 * The property containing the reference.
+		 */
+		property_ptr prop;
+		/**
+		 * The property value that contains the reference.
+		 */
+		property_value &val;
+	};
+	/**
 	 * A collection of property values that refer to phandles.  These will
 	 * be replaced by the value of the phandle property in their
 	 * destination.
 	 */
-	std::vector<property_value*> phandles;
+	std::vector<fixup> fixups;
+	/**
+	 * The locations of all of the values that are supposed to become phandle
+	 * references, but refer to things outside of this file.  
+	 */
+	std::vector<std::reference_wrapper<fixup>> unresolved_fixups;
 	/**
 	 * The names of nodes that target phandles.
 	 */
@@ -733,6 +787,10 @@ class device_tree
 	 */
 	uint32_t blob_padding;
 	/**
+	 * Is this tree a plugin?
+	 */
+	bool is_plugin;
+	/**
 	 * Visit all of the nodes recursively, and if they have labels then add
 	 * them to the node_paths and node_names vectors so that they can be
 	 * used in resolving cross references.  Also collects phandle
@@ -772,6 +830,11 @@ class device_tree
 	void write(int fd);
 	public:
 	/**
+	 * Should we write the __symbols__ node (to allow overlays to be linked
+	 * against this blob)?
+	 */
+	bool write_symbols = false;
+	/**
 	 * Returns the node referenced by the property.  If this is a tree that
 	 * is in source form, then we have a string that we can use to index
 	 * the cross_references array and so we can just look that up.  
diff --git a/usr.bin/dtc/input_buffer.cc b/usr.bin/dtc/input_buffer.cc
index 2f02426c0dc8e..a73e2b0945df6 100644
--- a/usr.bin/dtc/input_buffer.cc
+++ b/usr.bin/dtc/input_buffer.cc
@@ -102,7 +102,7 @@ struct stream_input_buffer : public dtc::input_buffer
 	stream_input_buffer();
 };
 
-mmap_input_buffer::mmap_input_buffer(int fd, std::string &&filename)
+mmap_input_buffer::mmap_input_buffer(int fd, string &&filename)
 	: input_buffer(0, 0), fn(filename)
 {
 	struct stat sb;
@@ -216,6 +216,7 @@ text_input_buffer::handle_include()
 		parse_error("Expected quoted filename");
 		return;
 	}
+	auto loc = location();
 	string file = parse_to('"');
 	consume('"');
 	if (!reallyInclude)
@@ -243,7 +244,7 @@ text_input_buffer::handle_include()
 	}
 	if (!include_buffer)
 	{
-		parse_error("Unable to locate input file");
+		loc.report_error("Unable to locate input file");
 		return;
 	}
 	input_stack.push(std::move(include_buffer));
@@ -1214,7 +1215,7 @@ input_buffer::buffer_for_file(const string &path, bool warn)
 		close(source);
 		return 0;
 	}
-	std::unique_ptr<input_buffer> b(new mmap_input_buffer(source, std::string(path)));
+	std::unique_ptr<input_buffer> b(new mmap_input_buffer(source, string(path)));
 	close(source);
 	return b;
 }
diff --git a/usr.bin/dtc/string.cc b/usr.bin/dtc/string.cc
index afc7bfc456d61..8891556a54935 100644
--- a/usr.bin/dtc/string.cc
+++ b/usr.bin/dtc/string.cc
@@ -31,6 +31,7 @@
  */
 
 #include <string>
+#include <functional>
 #include <cstdio>
 #include <cstdlib>
 #include <ctype.h>
@@ -121,28 +122,28 @@ push_string(byte_buffer &buffer, const string &s, bool escapes)
 	}
 }
 
-std::string dirname(const string &s)
+namespace {
+string
+dirbasename(std::function<char*(char*)> fn, const string &s)
 {
 	if (s == string())
 	{
 		return string();
 	}
-	char *str = strdup(s.c_str());
-	string dn(::dirname(str));
-	free(str);
+	std::unique_ptr<char, decltype(free)*> str = {strdup(s.c_str()), free};
+	string dn(fn(str.get()));
 	return dn;
 }
+}
 
-std::string basename(const string &s)
+string dirname(const string &s)
 {
-	if (s == string())
-	{
-		return string();
-	}
-	char *str = strdup(s.c_str());
-	string bn(::basename(str));
-	free(str);
-	return bn;
+	return dirbasename(::dirname, s);
+}
+
+string basename(const string &s)
+{
+	return dirbasename(::basename, s);
 }
 } // namespace dtc