Revision 9th of November

adddata.py

@@ -0,0 +1,28 @@
+from pathlib import Path
+import json
+
+dataset_path = Path('dataset')
+dataset_info = dataset_path / 'dataset.json'
+
+with open(dataset_info, 'r') as f:
+    dataset = json.load(f)
+
+additionaldata = {'apssouza22/java-microservice': {'s': 13, 'e': 2, 'i': 34, 'a': 100, 't': 149, 'l': 5283, 'tech': {'Kafka', 'Docker', 'ZooKeeper', 'Docker Compose', 'Hystrix', 'Spring OAuth', 'Nginx', 'Spring Config', 'Eureka', 'Spring Admin', 'Maven', 'Logstash'}}, 'callistaenterprise/blog-microservices': {'s': 15, 'e': 2, 'i': 32, 'a': 126, 't': 175, 'l': 2786, 'tech': {'RabbitMQ', 'Gradle', 'Zipkin', 'Docker Compose', 'Hystrix', 'Ribbon', 'Elasticsearch', 'Spring OAuth', 'Spring Config', 'Turbine', 'Eureka', 'Zuul', 'Logstash', 'Kibana'}}, 'fernandoabcampos/spring-netflix-oss-microservices': {'s': 9, 'e': 2, 'i': 24, 'a': 75, 't': 110, 'l': 1644, 'tech': {'Docker', 'RabbitMQ', 'Turbine', 'Hystrix', 'Spring Config', 'Eureka', 'Zuul', 'Maven'}}, 'georgwittberger/apache-spring-boot-microservice-example': {'s': 4, 'e': 1, 'i': 6, 'a': 18, 't': 29, 'l': 604, 'tech': {'Maven', 'Apache httpd'}}, 'mudigal-technologies/microservices-sample': {'s': 14, 'e': 1, 'i': 34, 'a': 115, 't': 164, 'l': 14527, 'tech': {'Docker', 'RabbitMQ', 'Docker Compose', 'Weave Scope', 'Elasticsearch', 'Nginx', 'Consul', 'Zuul', 'Logstash', 'Kibana'}}, 'spring-petclinic/spring-petclinic-microservices': {'s': 10, 'e': 2, 'i': 28, 'a': 89, 't': 129, 'l': 3990, 'tech': {'Docker', 'Grafana', 'Zipkin', 'Docker Compose', 'Hystrix', 'Prometheus', 'Ribbon', 'Spring Config', 'Spring Gateway', 'Eureka', 'Spring Admin', 'Maven'}}, 'sqshq/piggymetrics': {'s': 14, 'e': 3, 'i': 37, 'a': 192, 't': 246, 'l': 9977, 'tech': {'Docker', 'RabbitMQ', 'Spring OAuth', 'Hystrix', 'Ribbon', 'Turbine', 'Spring Config', 'Eureka', 'Zuul', 'Maven'}}, 'anilallewar/microservices-basics-spring-boot': {'s': 10, 'e': 2, 'i': 29, 'a': 108, 't': 149, 'l': 4245, 'tech': {'Docker', 'Gradle', 'Zipkin', 'Docker Compose', 'Hystrix', 'Ribbon', 'Spring OAuth', 'Turbine', 'Spring Config', 'Eureka', 'Zuul'}}, 'ewolff/microservice': {'s': 6, 'e': 1, 'i': 13, 'a': 46, 't': 66, 'l': 3117, 'tech': {'Turbine', 'Hystrix', 'Ribbon', 'Eureka', 'Zuul', 'Maven'}}, 'ewolff/microservice-kafka': {'s': 7, 'e': 1, 'i': 12, 'a': 56, 't': 76, 'l': 2979, 'tech': {'Kafka', 'ZooKeeper', 'Docker', 'Docker Compose', 'Apache httpd'}}, 'jferrater/Tap-And-Eat-MicroServices': {'s': 8, 'e': 1, 'i': 16, 'a': 63, 't': 88, 'l': 1484, 'tech': {'Docker', 'Docker Compose', 'Hystrix', 'Spring Config', 'Eureka', 'Maven'}}, 'koushikkothagal/spring-boot-microservices-workshop': {'s': 4, 'e': 1, 'i': 6, 'a': 26, 't': 37, 'l': 638, 'tech': {'Maven', 'Eureka'}}, 'mdeket/spring-cloud-movie-recommendation': {'s': 6, 'e': 5, 'i': 18, 'a': 93, 't': 122, 'l': 1800, 'tech': {'Zipkin', 'Hystrix', 'Ribbon', 'Spring Config', 'Eureka', 'Zuul', 'Maven'}}, 'piomin/sample-spring-oauth2-microservices': {'s': 5, 'e': 3, 'i': 13, 'a': 78, 't': 99, 'l': 1028, 'tech': {'Spring OAuth', 'Eureka', 'Ribbon', 'Zuul'}}, 'shabbirdwd53/springboot-microservice': {'s': 7, 'e': 2, 'i': 18, 'a': 65, 't': 92, 'l': 879, 'tech': {'Zipkin', 'Hystrix', 'Spring Config', 'Spring Gateway', 'Eureka', 'Maven'}}, 'rohitghatol/spring-boot-microservices': {'s': 8, 'e': 3, 'i': 26, 'a': 100, 't': 137, 'l': 2328, 'tech': {'Gradle', 'Docker Compose', 'Hystrix', 'Ribbon', 'Spring OAuth', 'Spring Config', 'Eureka', 'Zuul'}}, 'yidongnan/spring-cloud-netflix-example': {'s': 9, 'e': 1, 'i': 30, 'a': 81, 't': 121, 'l': 1182, 'tech': {'Docker', 'RabbitMQ', 'Gradle', 'Zipkin', 'Docker Compose', 'Hystrix', 'Ribbon', 'Turbine', 'Spring Config', 'Eureka', 'Spring Admin', 'Zuul'}}}
+
+technologies = set()
+for repo, addinfo in additionaldata.items():
+    addinfo['tech'] = list(sorted(addinfo['tech'], key=str.casefold))
+    technologies.update(addinfo['tech'])
+    success = False
+    for model_id, info in dataset.items():
+        if info['slug'].lower() == repo.lower():
+            success = True
+            print(f"Found repo {repo}")
+            info.update(addinfo)
+            print(json.dumps(info))
+            break
+    if not success:
+        raise Exception(f"Repo not found: {repo}!")
+
+with open(dataset_path / 'dataset.json', 'w') as f:
+    json.dump(dataset, f)

createreadmes.py

@@ -220,4 +220,4 @@ def main():
     save_dataset(dataset)
 
 if __name__ == '__main__':
-    main()
+    main()

extract.py

@@ -0,0 +1,20 @@
+import re
+from pathlib import Path
+
+def extract_source(name):
+    model = Path(name) / Path(f"{name}.py")
+    text = model.read_text()
+    match = re.search(r"^#\s*sources?\s*:\s*(?:\n#\s*-\s*)?(.+?)\s*$", text, re.MULTILINE | re.IGNORECASE)
+    if not match:
+        print(text)
+        raise Exception(f"No match found in {name}!")
+    source = match.group(1)
+    return source.strip('#- ')
+
+def main():
+    path = Path('.')
+    for model in (i for i in path.iterdir() if i.is_dir()):
+        print(model, '|', extract_source(model))
+
+if __name__ == '__main__':
+    main()