90 lines
3.5 KiB
Python
90 lines
3.5 KiB
Python
"""
|
|
Script to iterate over histogram data and insert it into the database.
|
|
This is a stub for future development.
|
|
"""
|
|
import os
|
|
import re
|
|
import numpy as np
|
|
import sqlalchemy
|
|
from sqlalchemy import text, exc
|
|
|
|
# Database URL from alembic.ini
|
|
DATABASE_URL = "postgresql+psycopg2://postgres:pgRocks@localhost:5432/testdb"
|
|
BASE_DIR = '/home/kfl/dev/git/avdb-py-v1'
|
|
|
|
def insert_histogram_data():
|
|
"""
|
|
Iterate over histogram data for levels 3-5 and insert it into the database.
|
|
This is a stub and will be developed over several prompts.
|
|
"""
|
|
engine = sqlalchemy.create_engine(DATABASE_URL)
|
|
|
|
# Placeholder for histogram data directory
|
|
img_folder = "img"
|
|
inserted_count = 0
|
|
skipped_count = 0
|
|
duplicate_count = 0
|
|
|
|
# Iterate over histogram data files (stub)
|
|
for root, dirs, files in os.walk(img_folder):
|
|
for file in files:
|
|
if file.lower().endswith('.mmap'):
|
|
histogram_file_path = os.path.join(BASE_DIR, root, file)
|
|
print(histogram_file_path)
|
|
level = 0
|
|
m = re.match(r'.*(\d{1})\.jpeg.histos.mmap', histogram_file_path)
|
|
if m:
|
|
level = int(m.group(1))
|
|
if level in (3,4,5):
|
|
# Extract histogram metadata (stub)
|
|
try:
|
|
histogram_values = [] # Placeholder
|
|
|
|
gridsplit = 0
|
|
if level == 3:
|
|
gridsplit = 8
|
|
elif level == 4:
|
|
gridsplit = 4
|
|
elif level == 5:
|
|
gridsplit = 2
|
|
|
|
data = np.memmap(
|
|
histogram_file_path,
|
|
dtype="float32",
|
|
mode='r+',
|
|
shape=((gridsplit+1)**2,48)
|
|
)
|
|
print(data.shape)
|
|
for row in data:
|
|
print(row)
|
|
|
|
# # Insert into the database (stub)
|
|
# with engine.connect() as connection:
|
|
# query = text(
|
|
# """
|
|
# INSERT INTO histogram_table (level, histogram_values)
|
|
# VALUES (:level, :histogram_values)
|
|
# """
|
|
# )
|
|
# connection.execute(
|
|
# query,
|
|
# {"level": level, "histogram_values": histogram_values}
|
|
# )
|
|
# connection.commit()
|
|
# inserted_count += 1
|
|
# print(f"Inserted histogram data for level {level}")
|
|
except exc.IntegrityError as e:
|
|
# Handle duplicate key violations
|
|
print(f"Duplicate histogram entry skipped for {histogram_file_path}: {e}")
|
|
duplicate_count += 1
|
|
except Exception as e:
|
|
print(f"Failed to insert histogram data for {histogram_file_path}: {e}")
|
|
skipped_count += 1
|
|
|
|
print(f"\nTotal histogram entries inserted: {inserted_count}")
|
|
print(f"Total duplicates skipped: {duplicate_count}")
|
|
print(f"Total entries skipped due to errors: {skipped_count}")
|
|
|
|
if __name__ == "__main__":
|
|
insert_histogram_data()
|