initial commit of new version

2026-10-04 18:24:18 -04:00
parent 994fac96e0
commit 57220375a3
43 changed files with 3429 additions and 1 deletions
@@ -0,0 +1,89 @@
+"""
+Script to iterate over histogram data and insert it into the database.
+This is a stub for future development.
+"""
+import os
+import re
+import numpy as np
+import sqlalchemy
+from sqlalchemy import text, exc
+
+# Database URL from alembic.ini
+DATABASE_URL = "postgresql+psycopg2://postgres:pgRocks@localhost:5432/testdb"
+BASE_DIR = '/home/kfl/dev/git/avdb-py-v1'
+
+def insert_histogram_data():
+    """
+    Iterate over histogram data for levels 3-5 and insert it into the database.
+    This is a stub and will be developed over several prompts.
+    """
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    
+    # Placeholder for histogram data directory
+    img_folder = "img"
+    inserted_count = 0
+    skipped_count = 0
+    duplicate_count = 0
+    
+    # Iterate over histogram data files (stub)
+    for root, dirs, files in os.walk(img_folder):
+        for file in files:
+            if file.lower().endswith('.mmap'):
+                histogram_file_path = os.path.join(BASE_DIR, root, file)
+                print(histogram_file_path)
+                level = 0
+                m = re.match(r'.*(\d{1})\.jpeg.histos.mmap', histogram_file_path)
+                if m:
+                    level = int(m.group(1))
+                if level in (3,4,5):
+                    # Extract histogram metadata (stub)
+                    try:
+                        histogram_values = []  # Placeholder
+
+                        gridsplit = 0
+                        if level == 3:
+                            gridsplit = 8
+                        elif level == 4:
+                            gridsplit = 4
+                        elif level == 5:
+                            gridsplit = 2
+
+                        data = np.memmap(
+                            histogram_file_path,
+                            dtype="float32",
+                            mode='r+',
+                            shape=((gridsplit+1)**2,48)
+                        )
+                        print(data.shape)
+                        for row in data:
+                            print(row)
+                    
+                        # # Insert into the database (stub)
+                        # with engine.connect() as connection:
+                        #     query = text(
+                        #         """
+                        #         INSERT INTO histogram_table (level, histogram_values)
+                        #         VALUES (:level, :histogram_values)
+                        #         """
+                        #     )
+                        #     connection.execute(
+                        #         query,
+                        #         {"level": level, "histogram_values": histogram_values}
+                        #     )
+                        #     connection.commit()
+                        #     inserted_count += 1
+                        #     print(f"Inserted histogram data for level {level}")
+                    except exc.IntegrityError as e:
+                        # Handle duplicate key violations
+                        print(f"Duplicate histogram entry skipped for {histogram_file_path}: {e}")
+                        duplicate_count += 1
+                    except Exception as e:
+                        print(f"Failed to insert histogram data for {histogram_file_path}: {e}")
+                        skipped_count += 1
+        
+    print(f"\nTotal histogram entries inserted: {inserted_count}")
+    print(f"Total duplicates skipped: {duplicate_count}")
+    print(f"Total entries skipped due to errors: {skipped_count}")
+
+if __name__ == "__main__":
+    insert_histogram_data()