Creating a script that helps with finding dependencies, especially transitive dependencies of BUCK artifacts.
Change-Id: I5e877c07b8027a57f88b583c11cf377cc4f25b0a
diff --git a/tools/dev/bin/onos-dep-discover b/tools/dev/bin/onos-dep-discover
new file mode 100755
index 0000000..489a23f
--- /dev/null
+++ b/tools/dev/bin/onos-dep-discover
@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+"""
+This is a scraper based series of scripts designed to retrieve the complete set
+of deps of a given artifact. This script is brittle, it will USUALLY fetch the
+set of dependencies however it may fail in certain cases, including on pages
+with unusual url table structures.
+
+In its current configuration it will list the dependencies that were officially
+supported at release, it will not note any updates whatsoever.
+"""
+import urllib.request
+import urllib.response
+
+
+fringe = set()
+already_checked = set()
+human_readable_deps = set()
+
+url_prefix = "https://mvnrepository.com/artifact/"
+
+compile_deps_string = "<h2>Compile Dependencies ("
+
+table_open_string = "<tbody"
+table_close_string = "</tbody>"
+
+table_row_open_string = "<tr>"
+table_row_close_string = "</tr>"
+
+table_row_cell_open_string = "<td"
+table_row_cell_close_string = "</td>"
+
+unique_identified_prefix = "<a class=\"vbtn release\" href=\"/artifact/"
+
+
+# url MUST be of the form https://mvnrepository.com/artifact/*organization_id*/*artifact_id*/*version_id*
+# This method takes a starting point url and compiles and returns a list of all the dependencies of that artifact
+def get_deps_for_artifact(url):
+ if url_prefix != url[:len(url_prefix)]:
+ raise ValueError("The url must begin with a valid https address for the maven central repository.")
+ stripped_url = url.replace(url_prefix, "")
+ fringe.add(stripped_url)
+ while len(fringe) != 0:
+ to_process = fringe.pop()
+ if to_process in already_checked:
+ continue
+ already_checked.add(to_process)
+ for dep in get_deps_from_page(url_prefix + to_process):
+ if dep not in already_checked and dep not in fringe:
+ fringe.add(dep)
+ return already_checked
+
+
+# adds all dependencies on the specified page that have not been previously seen to the fringe
+# page is expected to be a valid url
+def get_deps_from_page(url):
+ page_deps = set()
+ # get the string version of this site
+ html_string = urllib.request.urlopen(url).read().decode()
+
+ # Determine how many compile deps
+ start_index = html_string.find(compile_deps_string) + len(compile_deps_string)
+ end_index = html_string.find(")</h2>", start_index)
+ compile_deps_count = int(html_string[start_index: end_index])
+ # Get the compile deps if any
+ if compile_deps_count != 0:
+ table_open_index = html_string.find(table_open_string, end_index)
+ table_close_index = html_string.find(table_close_string, table_open_index) + len(table_close_string)
+ compile_deps = get_deps_from_table(html_string, table_open_index, table_close_index, compile_deps_count)
+ page_deps.update(compile_deps)
+
+ return page_deps
+
+#Table is expected to be from the "<tbody>" tag to the "</tbody>" tag.
+def get_deps_from_table(html_string, table_open_index, table_close_index, expected_count):
+ table_deps = set()
+ start_index = html_string.find(table_row_open_string, table_open_index, table_close_index)
+
+ while start_index != -1:
+ end_index = html_string.find(table_row_close_string, start_index, table_close_index)\
+ + len(table_row_close_string)
+ row_dep = get_dep_from_row(html_string, start_index, end_index)
+ if row_dep != None:
+ table_deps.add(row_dep)
+ start_index = html_string.find(table_row_open_string, end_index, table_close_index)
+
+ return table_deps
+
+#The row is expected to be from "<tr" tag to "</tr>" tag, it is expected to contain 5 pairs of matched
+#"<td...></td>" tags, the fourth such pair will contain the desired information,
+# the 5th will contain updated versions
+def get_dep_from_row(html_string, row_start_index, row_end_index):
+ start_index = row_start_index
+ end_index = row_end_index
+ start_index = html_string.find(table_row_cell_open_string, start_index, row_end_index)
+ end_index = html_string.find(table_row_cell_close_string, start_index, row_end_index)\
+ + len(table_row_cell_close_string)
+ #set the indecies for the fourth "<td>" element
+ for i in range(3):
+ start_index = html_string.find(table_row_cell_open_string, end_index, row_end_index)
+ end_index = html_string.find(table_row_cell_close_string, start_index, row_end_index)\
+ + len(table_row_cell_close_string)
+
+ return get_dep_from_cell(html_string, start_index, end_index)
+
+def get_dep_from_cell(html_string, cell_start_index, cell_end_index):
+ start_index = html_string.find(unique_identified_prefix, cell_start_index, cell_end_index)\
+ + len(unique_identified_prefix)
+ end_index = html_string.find("\"", start_index, cell_end_index)
+ if start_index == (len(unique_identified_prefix) - 1):
+ return None
+ return html_string[start_index:end_index]
+
+# Produces the dependency set but returns them in mvn coord style
+def get_mvn_coordinates_deps(url):
+ mvn_coords = set()
+ for elem in get_deps_for_artifact(url):
+ artifact_start_index = elem.find("/", 0)
+ org_id = elem[:artifact_start_index]
+ version_start_index = elem.find("/", artifact_start_index + 1)
+ artifact_id = elem[artifact_start_index + 1: version_start_index]
+ version = elem[version_start_index + 1:]
+ mvn_coords.add("mvn:" + org_id + ":" +artifact_id + ":" + version)
+
+ return mvn_coords
+
+
+# Prints out the complete set of deps for the specified package(s)
+def print_collection(lst):
+ for elem in lst:
+ print(elem)
+ return
+
+
+def main():
+ url = input("Please enter the url of the repo whose dependencies you would like?" +
+ "\n(this should be a fully qualified url\nex: https://mvnrepository." +
+ "com/artifact/com.google.guava/guava/19.0)")
+ mvn_coords = input("Would you like maven coordinate output, enter 'y' for yes? (alternately url style paths will be provided)")
+ if mvn_coords == "y" or mvn_coords == "Y":
+ print_collection(get_mvn_coordinates_deps(url))
+ else:
+ print_collection(get_deps_for_artifact(url))
+
+if __name__ == '__main__':
+ main()