Initial db importer

db-importer/db_importer/__main__.py

@@ -0,0 +1,38 @@
+
+import json
+import os
+import os.path
+import sys
+
+import pymongo
+from pymongo import MongoClient
+
+DB = "testdb"
+COLLECTION = "files"
+
+def main():
+    path = sys.argv[1]
+    mongo = MongoClient(f"mongodb://root:toor@apps.kunc.me/{DB}?authSource=admin")
+    print(f"Loading files from path: {path}")
+    items = []
+    collections = {}
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            path = os.path.join(root, file)
+            with open(path) as f:
+                obj = json.load(f)
+                obj["__src_file"] = path
+                items.append(obj)
+                obj_type = obj.get("type",None)
+                if obj_type is not None:
+                    if obj_type not in collections:
+                        collections[obj_type] = []
+                    collections[obj_type].append(obj)
+    print("Inserting all data")
+    mongo[DB][COLLECTION].insert_many(items)
+    for collection in collections:
+        print(f"Inserting collection: {collection}")
+        mongo[DB][collection].insert_many(collections[collection])
+
+if __name__ == "__main__":
+    main()