tms/avdb-py
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit of new version

Browse Source
This commit is contained in:
tms
2026-10-04 18:24:18 -04:00
parent 994fac96e0
commit 57220375a3
43 changed files with 3429 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
avdb/db.py
+679
View File
@@ -0,0 +1,679 @@
import os
import re
import json
import uuid
import csv
from typing import List, Tuple
import psycopg2
from psycopg2 import extras
import numpy as np
import cv2 as cv
from avdb.config import LEVELS_CONFIG
BATCH_SIZE=200
class DBConnection:
def __init__(self, dbname='testdb'):
try:
self._connection = psycopg2.connect(
user="postgres",
password='pgRocks',
host="127.0.0.1",
port="5432",
database=dbname
)
print(f"\nconnection: {self._connection}\n\n")
self._connection.autocommit = True
self.dbname = dbname
# Create a cursor to execute queries
self.cursor = self._connection.cursor()
# Print PostgreSQL details
print("PostgreSQL server information")
print(self._connection.get_dsn_parameters(), "\n")
# Executing a SQL query
self.cursor.execute("SELECT version();")
# Fetch result
record = self.cursor.fetchone()
print("You are connected to - ", record, "\n")
except (Exception, psycopg2.Error) as error:
print("Error while connecting to PostgreSQL", error)
def get_cursor(self):
return self._connection.cursor()
def get_connection(self):
return self._connection
def test_db_connection():
"""
Test if the database is active and available by attempting a connection and a simple query.
Returns:
bool: True if the database connection is successful, False otherwise.
"""
try:
# Attempt to establish a connection
conn = DBConnection('testdb')
cursor = conn.get_cursor()
# Execute a simple query to verify the connection
cursor.execute("SELECT 1;")
result = cursor.fetchone()
if result[0] == 1:
print("Database connection test successful!")
return True
else:
print("Database connection test failed.")
return False
except (Exception, psycopg2.Error) as error:
print(f"Error while testing database connection: {error}")
return False
class DBOps:
def __init__(self, connection):
# self._dbname = connection.dbname
self._connection = connection
def insert_image(
self,
img_path: str,
img_name: str,
width: int,
height: int,
img_id= None
) -> int:
"""
Insert a new image record into the images table.
Args:
img_path (str): Path to the image file.
img_name (str): Name of the image file.
width (int): Width of the image in pixels.
height (int): Height of the image in pixels.
Returns:
int: The ID of the newly inserted image record.
"""
print(f"Inserting image: {img_path},{img_id}")
query_str = """
INSERT INTO avdb_image (img_path, img_name, width, height)
VALUES (%s, %s, %s, %s)
RETURNING img_id;
"""
with self._connection.get_cursor() as cursor:
cursor.execute(query_str, (img_path, img_name, width, height))
img_id_out = cursor.fetchone()[0]
print(f"Warning if true: {img_id != img_id_out}")
return img_id_out
def insert_histogram_data(
self,
img_id: int,
level: int,
tilex: int,
tiley: int,
x: int,
y: int,
x0: int,
y0: int,
w: int,
h: int,
histogram_data: List[float]
) -> int:
"""
Insert histogram data into the histogramfeatures table.
Args:
img_id (int): ID of the image.
level (int): Pyramid level.
tilex (int): Logical tile x-coordinate.
tiley (int): Logical tile y-coordinate.
x (int): Upper left x-coordinate (in original resolution).
y (int): Upper left y-coordinate (in original resolution).
x0 (int): Upper left x-coordinate (in down-sampled resolution).
y0 (int): Upper left y-coordinate (in down-sampled resolution).
w (int): Width of the histogram region.
h (int): Height of the histogram region.
histogram_data (np.ndarray): Histogram data as a numpy array.
[tile_data: NULL.]
Returns:
int: The ID of the newly inserted histogram record.
"""
query_str = """
INSERT INTO pyramiddata (
img_id, level, tilex, tiley, x, y, x0, y0, w, h, hash, histogram
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING pyrdata_id;
"""
hsh = uuid.uuid4()
with self._connection.cursor() as cursor:
cursor.execute(
query_str,
(
img_id,
level,
tilex,
tiley,
x,
y,
x0,
y0,
w,
h,
str(hsh),
histogram_data
)
)
try:
pyrdata_id = cursor.fetchone()[0]
except Exception as e:
print(f"EXEPTION: {e}")
return pyrdata_id
def insert_block_data(
self,
img_id: int,
level: int,
tilex: int,
tiley: int,
x: int,
y: int,
x0: int,
y0: int,
w: int,
h: int,
block_data: np.ndarray
) -> int:
"""
Insert block data into the tilefeatures table.
Args:
img_id (int): ID of the image.
level (int): Pyramid level.
tilex (int): Logical tile x-coordinate.
tiley (int): Logical tile y-coordinate.
x (int): Center x-coordinate (in original resolution).
y (int): Center y-coordinate (in original resolution).
x0 (int): Center x-coordinate (in down-sampled resolution).
y0 (int): Center y-coordinate (in down-sampled resolution).
w (int): Width of the tile.
h (int): Height of the tile.
block_data (np.ndarray): Tile data as a numpy array.
[histogram_data: NULL.]
Returns:
int: The ID of the newly inserted tile record.
"""
query_str = """
INSERT INTO pyramiddata (
img_id, level, tilex, tiley, x, y, x0, y0, w, h, hash, block_data
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
RETURNING pyrdata_id;
"""
hsh = uuid.uuid4()
with self._connection.get_cursor() as cursor:
cursor.execute(
query_str,
(
img_id,
level,
tilex,
tiley,
x,
y,
x0,
y0,
w,
h,
str(hsh),
block_data
)
)
pyrdata_id = cursor.fetchone()[0]
return pyrdata_id
def insert_tile_data(self, rows: List[Tuple]) -> None:
"""
Insert tile data into the pyramiddata table using batch insertion.
Args:
rows (List[Tuple]): List of tuples containing tile data to insert.
"""
query_str = """
INSERT INTO pyramiddata (
img_id, level, tilex, tiley, x, y, x0, y0, w, h, hash, tile_data
)
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
"""
with self._connection.cursor() as cursor:
# Use execute_batch for efficient batch insertion
print(f"Inserting {len(rows)} rows into pyramid {rows[:1]}")
extras.execute_batch(cursor, query_str, rows)
self._connection.commit()
def load_memmapped_histogram_data(self, file_path: str, rows=81):
"""
Load memory-mapped histogram data from a file.
Args:
file_path (str): Path to the file containing the histogram data.
Returns:
np.ndarray: The loaded histogram data as a numpy array.
"""
return np.memmap(file_path, mode='r+', dtype='float32', shape=(rows,48))
def load_memmapped_tile_data(self, file_path: str):
"""
Load memory-mapped tile data from a file.
Args:
file_path (str): Path to the file containing the tile data.
Returns:
np.ndarray: The loaded tile data as a numpy array.
"""
return np.memmap(file_path, mode='r+', dtype=np.uint16)
def load_memmapped_block_data(self, file_path: str):
"""
Load memory-mapped tile data from a file.
Args:
file_path (str): Path to the file containing the tile data.
Returns:
np.ndarray: The loaded tile data as a numpy array.
"""
return np.memmap(file_path, mode='r+', dtype=np.uint16)
def get_image_by_id(self, img_id: int) -> Tuple:
"""
Retrieve an image record by its ID.
Args:
img_id (int): ID of the image to retrieve.
Returns:
tuple: A tuple representing the image record.
"""
query_str = "SELECT * FROM images WHERE img_id = %s;"
with self._connection.cursor() as cursor:
cursor.execute(query_str, (img_id,))
return cursor.fetchone()
def get_pyramid_metadata_by_image_id(self, img_id: int) -> List[Tuple]:
"""
Retrieve all pyramid metadata records associated with an image.
Args:
img_id (int): ID of the image.
Returns:
list: A list of tuples representing the pyramid metadata records.
"""
query_str = "SELECT * FROM pyramidmetadata WHERE img_id = %s;"
with self._connection.cursor() as cursor:
cursor.execute(query_str, (img_id,))
return cursor.fetchall()
def get_pyramid_metadata_by_id(self, pyramid_id: int) -> Tuple:
"""
Retrieve a pyramid metadata record by its ID.
Args:
pyramid_id (int): ID of the pyramid metadata record.
Returns:
tuple: A tuple representing the pyramid metadata record.
"""
query_str = "SELECT * FROM pyramidmetadata WHERE pyramid_id = %s;"
with self._connection.cursor() as cursor:
cursor.execute(query_str, (pyramid_id,))
return cursor.fetchone()
def get_image_id_by_img_name(self, img_name: str) -> int:
"""
Retrieve the ID of an image by its name.
Args:
img_name (str): Name of the image file.
Returns:
int: The ID of the image, or None if not found.
"""
query_str = "SELECT img_id FROM avdb_image WHERE img_name = %s;"
with self._connection.cursor() as cursor:
cursor.execute(query_str, (img_name,))
result = cursor.fetchone()
return result[0] if result else None
def process_and_insert_histogram_data(
self,
img_id: int,
img_dims: Tuple[int, int],
level: int,
histogram_file: str,
grid_size: int = 9,
bin_size: int = 48
) -> None:
"""
Process and insert histogram data for a specific level of an image.
Args:
img_id (int): ID of the image.
img_dims (tuple): Dimensions of the image (width, height).
level (int): Pyramid level.
histogram_file (str): The memory-mapped histogram file.
grid_size (int): Size of the grid (e.g., 9 for 9x9 grid).
bin_size (int): Number of bins in the histogram (e.g., 48 for Lab histogram).
"""
if level<1:
raise ValueError("Level must be greater than 0")
# Calculate the downsampled resolution for this level
resolution = 2 ** level
# Load the memory-mapped histogram data
histogram_data = self.load_memmapped_histogram_data(histogram_file, grid_size**2)
# print(type(histogram_data))
print(f"histogram_data shape: {histogram_data.shape}")
img_w,img_h = img_dims
# print(list((histogram_data[0,:]).flatten()))
counter = 0
# Loop through each grid cell
for tilex in range(grid_size):
for tiley in range(grid_size):
# Calculate spatial coordinates (simplified for now)
# These calculations can be adjusted based on your specific requirements
x = tilex * (img_w // resolution)
y = tiley * (img_h // resolution)
x0 = tilex * (img_w // (resolution * grid_size ))
y0 = tiley * (img_h // (resolution * grid_size))
w = img_w // (resolution / 2)
h = img_h // (resolution / 2)
# Insert the histogram data into the database
# print(f"Inserting histogram data for tilex: {tilex}, tiley: {tiley}")
# print(f"x={x}, y={y}, x0={x0}, y0={y0}, w={w}, h={h}")
# print(type(list((histogram_data[counter,:]).flatten().tolist())[0]))
self.insert_histogram_data(
img_id=img_id,
level=level,
tilex=tilex,
tiley=tiley,
x=x,
y=y,
x0=x0,
y0=y0,
w=w,
h=h,
histogram_data=list((histogram_data[counter,:]).flatten().tolist())
)
counter += 1
def to_unsigned(self, signed_int):
return signed_int + (1 << 8) if signed_int < 0 else signed_int
def process_and_insert_tile_data_batched(
self,
img_id: int,
img_dims: Tuple[int, int],
level: int,
tile_file: str
) -> None:
"""
Process and insert tile data for a specific level of an image.
Args:
img_id (int): ID of the image.
img_dims (Tuple[int, int]): Dimensions of the image (width, height).
level (int): Pyramid level.
tile_file (str): The memory-mapped tile file.
grid_size (int): Size of the grid.
"""
# Calculate the downsampled resolution for this level
resolution = 2 ** level
img_w, img_h = img_dims
# Load the memory-mapped tile data
print(tile_file)
tile_data = self.load_memmapped_tile_data(tile_file)
print(f"img dims: {img_dims}")
print(f"tile data shape:{tile_data.shape}")
tilew = img_w // resolution
tileh = img_h // resolution
width_t = img_w // tilew
height_t = img_h // tileh
grid_size = 1
def gen_tiles():
counter = 0
# Loop through each tile
for tiley in range(tileh):
for tilex in range(tilew):
x0 = tilex * width_t
y0 = tiley * height_t
w = grid_size
h = 1
# print(f"{counter} - {level} - {tilex} - {tiley} - {tilew} - {tileh} - {width_t} - {height_t} - {x0} - {y0} - {w} - {h} -- {tile_data[(counter * 3):((counter+1) * 3)]}")
# Insert the tile data into the database
try:
tiledata = list(
(
np.r_[
tile_data[
(counter * 3):
((counter+1) * 3)
]
]
).flatten().tolist()
)
counter += 1
yield (
img_id,
level,
tilex,
tiley,
width_t,
height_t,
x0,
y0,
w,
h,
str(uuid.uuid4()),
list(map(self.to_unsigned,tiledata))
)
except Exception as e:
print(f"ERROR: {e}")
all_for_insertion = list(gen_tiles())
print(f"Inserting {len(all_for_insertion)} tiles")
self.insert_tile_data(all_for_insertion)
def process_and_insert_block_data(
self,
img_id: int,
img_dims: Tuple[int, int],
level: int,
block_file: str,
grid_size: int = 9
) -> None:
"""
Process and insert block data for a specific level of an image.
Args:
img_id (int): ID of the image.
img_dims (Tuple[int, int]): Dimensions of the image (width, height).
level (int): Pyramid level.
block_file (str): The memory-mapped block file.
grid_size (int): Size of the grid.
"""
# Calculate the downsampled resolution for this level
resolution = 2 ** level
img_w, img_h = img_dims
# if level == 6:
# row_size = 9
# elif level == 7:
# row_size = 5
# else:
if level not in (6, 7):
raise ValueError(f"Unsupported level ({level}). Level must be 6 or 7.")
# Load the memory-mapped block data
block_data = self.load_memmapped_block_data(block_file)
# print(f"img dims: {img_dims}")
# print(f"block data shape:{block_data.shape}")
tilew = img_w // resolution
tileh = img_h // resolution
width_t = img_w // tilew
height_t = img_h // tileh
# print(f"tilew: {tilew}, tileh: {tileh}, width_t: {width_t}, height_t: {height_t}")
counter = 0
# Loop through each block
for tilex in range(tilew-2):
for tiley in range(tileh-2):
x0 = tilex * width_t
y0 = tiley * height_t
w = grid_size
h = 1
# print(f"{counter} - {level} - {tilex} - {tiley} - {tilew} - {tileh} - {width_t} - {height_t} - {x0} - {y0} - {w} - {h}")
# Insert the block data into the database
# print(f"{counter * grid_size * 3} - {(counter+1) * grid_size * 3}")
try:
self.insert_block_data(
img_id=img_id,
level=level,
tilex=tilex,
tiley=tiley,
x=w,
y=h,
x0=x0,
y0=y0,
w=w,
h=h,
block_data=list(
(
np.r_[
block_data[
(counter * grid_size * 3):
((counter+1) * grid_size * 3)
]
# np.zeros((9 - grid_size) * 3)
]
).flatten().tolist()
)
)
except Exception as e:
print(f"ERROR: {e}")
pass
counter += 1
def extract_image_base(self, fullpath: str) -> str:
term = r'.*(IMG_\d{4}|TESTIMG).*'
m = re.match(term, fullpath)
if m is not None:
tkn = m.group(1)
return f"img/{tkn}/{tkn}.jpeg", tkn
raise ValueError(f"Could not extract image base from {fullpath}")
def process_image_analysis(
self,
img_path_in: str,
max_level: int = 8
) -> None:
"""
Process and insert all analysis data for an image across multiple levels.
Args:
img_id (int): ID of the image.
img_path (str): Full path.
max_level (int): max exponent to pyr down splits
"""
img_path, img_tkn = self.extract_image_base(img_path_in)
print(img_path)
# assumes image has been inserted into image table
img_id = self.get_image_id_by_img_name(os.path.basename(img_path))
print(f"IMAGE ID: {img_id}")
# insert image metadata
if not os.path.exists(img_path):
raise FileNotFoundError(f"{img_path} does not exist")
img = cv.imread(img_path)
width = img.shape[1]
height = img.shape[0]
print(f"Processing image {img_path} {width}x{height} {img_path}")
for level in range(max_level):
print(LEVELS_CONFIG[level])
if (LEVELS_CONFIG[level]["gridsplit"]) is not None:
data_path = f"{os.path.dirname(img_path)}/{img_path}.histos.mmap"
grid_size = LEVELS_CONFIG[level]["gridsplit"] + 1
# Process histogram data for this level
self.process_and_insert_histogram_data(
img_id=img_id,
img_dims=(width, height),
level=level,
histogram_file=data_path,
grid_size=grid_size
)
if (LEVELS_CONFIG[level]["tiles"]):
data_path = f"img/{img_tkn}/{img_tkn}.lab.pyr_{level}.jpeg.tiles.mmap"
print(data_path)
print("INSERTING TILE DATA")
# Process tile data for this level
self.process_and_insert_tile_data_batched(
img_id=img_id,
img_dims=(width, height),
level=level,
tile_file=data_path
)
if (LEVELS_CONFIG[level]["targetblocks"]) is not None:
data_path = f"{os.path.dirname(img_path)}/{img_path}.blocks.mmap"
print("INSERTING BLOCK DATA")
# Process block data for this level
self.process_and_insert_block_data(
img_id=img_id,
img_dims=(width, height),
level=level,
block_file=data_path
)
if __name__ == '__main__':
with DBConnection('testdb').get_connection() as db_conn:
db_ops = DBOps(db_conn)
db_ops.process_image_analysis('home/kfl/dev/git/avdb-py-v1/img/TESTIMG/TESTIMG.jpeg')