initial commit of new version
This commit is contained in:
tms
@@ -0,0 +1,59 @@
|
||||
"""
|
||||
Test module for the ColorFeaturesPyramid class.
|
||||
|
||||
This module contains unit tests to verify the functionality of the ColorFeaturesPyramid class,
|
||||
including initialization, down-sampling, and tile analysis.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import cv2 as cv
|
||||
import numpy as np
|
||||
|
||||
# from avdb.db import DBOps, DBConnection
|
||||
from avdb.color_features_pyramid import ColorFeaturesPyramid
|
||||
|
||||
|
||||
|
||||
def demo():
|
||||
"""
|
||||
Main function to demonstrate the usage of the ColorFeaturesPyramid class.
|
||||
|
||||
This function shows how to use the `_down_sample_and_analyze` method to perform
|
||||
down-sampling and tile analysis on an image.
|
||||
"""
|
||||
|
||||
for img_name in ["TESTIMG", "IMG_9997", "IMG_9998", "IMG_9999", "IMG_2327"]:
|
||||
|
||||
print(f"Processing image: {img_name}")
|
||||
|
||||
# Example image path and UUID
|
||||
img_path = f"img/{img_name}/{img_name}.jpeg"
|
||||
uuid = img_name
|
||||
|
||||
# Initialize the ColorFeaturesPyramid with custom parameters
|
||||
pyramid = ColorFeaturesPyramid(
|
||||
uuid=uuid,
|
||||
root_path=".",
|
||||
down_sampling=9,
|
||||
colorspace="lab"
|
||||
)
|
||||
|
||||
print("Starting down-sampling and analysis...")
|
||||
|
||||
# Run the down-sampling and analysis process
|
||||
results = list(pyramid._down_sample_and_analyze())
|
||||
|
||||
# Print the results for each level
|
||||
for level, (histograms, metadata) in results:
|
||||
imgw, imgh, tilesw, tilesh = metadata
|
||||
print(f"Level {level}:")
|
||||
print(f" Image dimensions: {imgw} x {imgh}")
|
||||
print(f" Tiles: {tilesw} x {tilesh}")
|
||||
print(f" Histograms shape: {histograms.shape}")
|
||||
print()
|
||||
|
||||
print("Down-sampling and analysis completed successfully.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
demo()
|
||||
@@ -0,0 +1,123 @@
|
||||
"""
|
||||
Test module for the ColorFeaturesPyramid class.
|
||||
|
||||
This module contains unit tests to verify the functionality of the ColorFeaturesPyramid class,
|
||||
including initialization, down-sampling, and tile analysis.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import numpy as np
|
||||
from avdb.db import DBOps, DBConnection
|
||||
from avdb.color_features_pyramid import ColorFeaturesPyramid
|
||||
|
||||
def demo():
|
||||
"""
|
||||
Main function to demonstrate the usage of the ColorFeaturesPyramid class.
|
||||
|
||||
This function shows how to use the `_down_sample_and_analyze` method to perform
|
||||
down-sampling and tile analysis on an image.
|
||||
"""
|
||||
# Example image path and UUID
|
||||
img_path = "img/TESTIMG.jpeg"
|
||||
uuid = "TESTIMG"
|
||||
|
||||
# Initialize the ColorFeaturesPyramid with custom parameters
|
||||
pyramid = ColorFeaturesPyramid(
|
||||
uuid=uuid,
|
||||
root_path=".",
|
||||
down_sampling=9,
|
||||
colorspace='lab'
|
||||
)
|
||||
|
||||
print("Starting down-sampling and analysis...")
|
||||
|
||||
# Run the down-sampling and analysis process
|
||||
results = list(pyramid._down_sample_and_analyze())
|
||||
|
||||
# Print the results for each level
|
||||
for level, (histograms, metadata) in results:
|
||||
imgw, imgh, tilesw, tilesh = metadata
|
||||
print(f"Level {level}:")
|
||||
print(f" Image dimensions: {imgw} x {imgh}")
|
||||
print(f" Tiles: {tilesw} x {tilesh}")
|
||||
print(f" Histograms shape: {histograms.shape}")
|
||||
print()
|
||||
|
||||
print("Down-sampling and analysis completed successfully.")
|
||||
|
||||
def analyze_all_images(limit=10, dryrun=True):
|
||||
"""
|
||||
Iterate over all images in the 'img/' folder and analyze them using the ColorFeaturesPyramid class.
|
||||
|
||||
This function processes each image in the 'img/' folder, initializes the ColorFeaturesPyramid,
|
||||
and performs down-sampling and tile analysis for each image.
|
||||
"""
|
||||
with DBConnection('testdb').get_connection() as db_conn:
|
||||
db_ops = DBOps(db_conn)
|
||||
|
||||
img_folder = "img"
|
||||
if not os.path.exists(img_folder):
|
||||
print(f"Error: The folder '{img_folder}' does not exist.")
|
||||
return
|
||||
|
||||
# Get all image files in the folder
|
||||
image_extensions = ('.jpg', '.jpeg')
|
||||
image_files = [
|
||||
f"{f}/{f}.jpeg" for f in os.listdir(img_folder)
|
||||
]
|
||||
|
||||
if not image_files:
|
||||
print(f"No images found in the folder '{img_folder}'.")
|
||||
return
|
||||
|
||||
print(f"Found {len(image_files)} images in the folder '{img_folder}'.")
|
||||
|
||||
for img_file in image_files[:limit]:
|
||||
img_path = os.path.join(img_folder, img_file)
|
||||
uuid = os.path.splitext(img_file.split('/')[1])[0] # Use the filename (without extension) as UUID
|
||||
|
||||
data_file_exists = os.path.exists(img_path.replace(".jpeg", ".lab.pyr_3.jpeg.tiles.mmap"))
|
||||
print(img_path.replace(".jpeg", ".lab.pyr_3.jpeg.tiles.mmap"))
|
||||
print(data_file_exists)
|
||||
# Check if the analysis files exist using level 7 as proxy
|
||||
if not data_file_exists:
|
||||
# img = cv.imread(img_path)
|
||||
# print(f"\nAnalyzing image: {img_path} - {os.path.basename(img_path)} - {len(image_files)} - {img.shape}")
|
||||
|
||||
# if not dryrun:
|
||||
# img_name = os.path.basename(img_path)
|
||||
# img_id_out = db_ops.insert_image(
|
||||
# img_path,
|
||||
# img_name,
|
||||
# img.shape[0],
|
||||
# img.shape[1]
|
||||
# )
|
||||
# print(f"Inserted image {i+1} {img_id_out}")
|
||||
|
||||
# Initialize the ColorFeaturesPyramid with custom parameters
|
||||
pyramid = ColorFeaturesPyramid(
|
||||
uuid=uuid,
|
||||
root_path=".",
|
||||
down_sampling=9,
|
||||
colorspace='lab'
|
||||
)
|
||||
|
||||
# Run the down-sampling and analysis process
|
||||
results = list(pyramid._down_sample_and_analyze())
|
||||
|
||||
# Print the results for each level
|
||||
for level, (histograms, metadata) in results:
|
||||
imgw, imgh, tilesw, tilesh = metadata
|
||||
print(f" Level {level}:")
|
||||
print(f" Image dimensions: {imgw} x {imgh}")
|
||||
print(f" Tiles: {tilesw} x {tilesh}")
|
||||
print(f" Histograms shape: {histograms.shape}\n\n")
|
||||
else:
|
||||
print(f"\nSkipping analysis file already exists:\n\t{img_file}\n")
|
||||
|
||||
print("\nAnalysis of all images completed successfully.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
# demo()
|
||||
analyze_all_images(limit=999, dryrun=False)
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Directory containing the JPEG images
|
||||
IMG_DIR="img"
|
||||
|
||||
# Check if the directory exists
|
||||
if [ ! -d "$IMG_DIR" ]; then
|
||||
echo "Directory $IMG_DIR does not exist."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Loop through each JPEG file in the directory
|
||||
for img_file in "$IMG_DIR"/*.jpg "$IMG_DIR"/*.jpeg; do
|
||||
# Check if the file exists (to handle cases where no files match the pattern)
|
||||
if [ -f "$img_file" ]; then
|
||||
# Extract the base name of the file (without extension)
|
||||
base_name=$(basename "$img_file" | cut -d. -f1)
|
||||
|
||||
# Create a folder with the same base name
|
||||
mkdir -p "$IMG_DIR/$base_name"
|
||||
|
||||
# Move the image file into the folder
|
||||
mv "$img_file" "$IMG_DIR/$base_name/"
|
||||
|
||||
echo "Moved $img_file to $IMG_DIR/$base_name/"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Done!"
|
||||
@@ -0,0 +1,89 @@
|
||||
"""
|
||||
Script to iterate over histogram data and insert it into the database.
|
||||
This is a stub for future development.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import numpy as np
|
||||
import sqlalchemy
|
||||
from sqlalchemy import text, exc
|
||||
|
||||
# Database URL from alembic.ini
|
||||
DATABASE_URL = "postgresql+psycopg2://postgres:pgRocks@localhost:5432/testdb"
|
||||
BASE_DIR = '/home/kfl/dev/git/avdb-py-v1'
|
||||
|
||||
def insert_histogram_data():
|
||||
"""
|
||||
Iterate over histogram data for levels 3-5 and insert it into the database.
|
||||
This is a stub and will be developed over several prompts.
|
||||
"""
|
||||
engine = sqlalchemy.create_engine(DATABASE_URL)
|
||||
|
||||
# Placeholder for histogram data directory
|
||||
img_folder = "img"
|
||||
inserted_count = 0
|
||||
skipped_count = 0
|
||||
duplicate_count = 0
|
||||
|
||||
# Iterate over histogram data files (stub)
|
||||
for root, dirs, files in os.walk(img_folder):
|
||||
for file in files:
|
||||
if file.lower().endswith('.mmap'):
|
||||
histogram_file_path = os.path.join(BASE_DIR, root, file)
|
||||
print(histogram_file_path)
|
||||
level = 0
|
||||
m = re.match(r'.*(\d{1})\.jpeg.histos.mmap', histogram_file_path)
|
||||
if m:
|
||||
level = int(m.group(1))
|
||||
if level in (3,4,5):
|
||||
# Extract histogram metadata (stub)
|
||||
try:
|
||||
histogram_values = [] # Placeholder
|
||||
|
||||
gridsplit = 0
|
||||
if level == 3:
|
||||
gridsplit = 8
|
||||
elif level == 4:
|
||||
gridsplit = 4
|
||||
elif level == 5:
|
||||
gridsplit = 2
|
||||
|
||||
data = np.memmap(
|
||||
histogram_file_path,
|
||||
dtype="float32",
|
||||
mode='r+',
|
||||
shape=((gridsplit+1)**2,48)
|
||||
)
|
||||
print(data.shape)
|
||||
for row in data:
|
||||
print(row)
|
||||
|
||||
# # Insert into the database (stub)
|
||||
# with engine.connect() as connection:
|
||||
# query = text(
|
||||
# """
|
||||
# INSERT INTO histogram_table (level, histogram_values)
|
||||
# VALUES (:level, :histogram_values)
|
||||
# """
|
||||
# )
|
||||
# connection.execute(
|
||||
# query,
|
||||
# {"level": level, "histogram_values": histogram_values}
|
||||
# )
|
||||
# connection.commit()
|
||||
# inserted_count += 1
|
||||
# print(f"Inserted histogram data for level {level}")
|
||||
except exc.IntegrityError as e:
|
||||
# Handle duplicate key violations
|
||||
print(f"Duplicate histogram entry skipped for {histogram_file_path}: {e}")
|
||||
duplicate_count += 1
|
||||
except Exception as e:
|
||||
print(f"Failed to insert histogram data for {histogram_file_path}: {e}")
|
||||
skipped_count += 1
|
||||
|
||||
print(f"\nTotal histogram entries inserted: {inserted_count}")
|
||||
print(f"Total duplicates skipped: {duplicate_count}")
|
||||
print(f"Total entries skipped due to errors: {skipped_count}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
insert_histogram_data()
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Script to iterate over all images in the img folder and insert their metadata into the avdb_image table.
|
||||
"""
|
||||
import os
|
||||
from PIL import Image
|
||||
import sqlalchemy
|
||||
from sqlalchemy import text, exc
|
||||
|
||||
# Database URL from alembic.ini
|
||||
DATABASE_URL = "postgresql+psycopg2://postgres:pgRocks@localhost:5432/testdb"
|
||||
|
||||
def get_image_metadata(image_path):
|
||||
"""Extract metadata (width, height) from an image."""
|
||||
with Image.open(image_path) as img:
|
||||
width, height = img.size
|
||||
return width, height
|
||||
|
||||
def insert_images_into_db():
|
||||
"""Iterate over all images in the img folder and insert their metadata into the avdb_image table."""
|
||||
engine = sqlalchemy.create_engine(DATABASE_URL)
|
||||
|
||||
# Iterate over all subdirectories in the img folder
|
||||
img_folder = "img"
|
||||
inserted_count = 0
|
||||
skipped_count = 0
|
||||
test_shim = 'TESTIMG.jpeg'
|
||||
duplicate_count = 0
|
||||
|
||||
for root, dirs, files in os.walk(img_folder):
|
||||
for file in files:
|
||||
# test nmode
|
||||
if file.endswith(test_shim):
|
||||
# if file.lower().endswith(('.jpg', '.jpeg')):
|
||||
image_path = os.path.join(root, file)
|
||||
img_name = file
|
||||
|
||||
# Extract width and height
|
||||
try:
|
||||
width, height = get_image_metadata(image_path)
|
||||
except Exception as e:
|
||||
print(f"Failed to extract metadata for {image_path}: {e}")
|
||||
skipped_count += 1
|
||||
continue
|
||||
|
||||
# Insert into the database
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
query = text(
|
||||
"""
|
||||
INSERT INTO avdb_image (img_path, img_name, width, height)
|
||||
VALUES (:img_path, :img_name, :width, :height)
|
||||
"""
|
||||
)
|
||||
connection.execute(
|
||||
query,
|
||||
{"img_path": image_path, "img_name": img_name, "width": width, "height": height}
|
||||
)
|
||||
connection.commit()
|
||||
inserted_count += 1
|
||||
print(f"Inserted: {img_name} (Width: {width}, Height: {height})")
|
||||
except exc.IntegrityError:
|
||||
# Handle duplicate key violations
|
||||
print(f"Duplicate entry skipped for {image_path}")
|
||||
duplicate_count += 1
|
||||
except Exception as e:
|
||||
print(f"Failed to insert {image_path} into the database: {e}")
|
||||
skipped_count += 1
|
||||
|
||||
print(f"\nTotal images inserted: {inserted_count}")
|
||||
print(f"Total duplicates skipped: {duplicate_count}")
|
||||
print(f"Total images skipped due to errors: {skipped_count}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
insert_images_into_db()
|
||||
@@ -0,0 +1,45 @@
|
||||
insert into pyramiddata_2tiles (
|
||||
pyrdata_id, img_id, level, tilex_a, tiley_a, x_a, y_a, x0_a, y0_a, w_a, h_a,
|
||||
tilex_b, tiley_b, x_b, y_b, x0_b, y0_b, w_b, h_b, hash, tile_data
|
||||
)
|
||||
select a.pyrdata_id as pyrdata_id
|
||||
,a.img_id as img_id
|
||||
,a.level as level
|
||||
,a.tilex as tilex_a
|
||||
,a.tiley as tiley_a
|
||||
,a.x as x_a
|
||||
,a.y as y_a
|
||||
,a.x0 as x0_a
|
||||
,a.y0 as y0_a
|
||||
,a.w as w_a
|
||||
,a.h as h_a
|
||||
,b.tilex as tilex_b
|
||||
,b.tiley as tilex_b
|
||||
,a.x as x_b
|
||||
,a.y as y_b
|
||||
,b.x0 as x0_b
|
||||
,b.y0 as y0_b
|
||||
,b.w as w_b
|
||||
,b.h as h_b
|
||||
,a.hash as hash
|
||||
,a.tile_data||b.tile_data as tile_data
|
||||
from (
|
||||
select *
|
||||
from pyramiddata
|
||||
where level >= 3
|
||||
and level <= 6
|
||||
--and img_id = 1
|
||||
and tile_data is not null
|
||||
) a
|
||||
JOIN (
|
||||
select *
|
||||
from pyramiddata
|
||||
where level >= 4
|
||||
and level <= 7
|
||||
--and img_id = 1
|
||||
and tile_data is not null
|
||||
) b
|
||||
on a.img_id = b.img_id
|
||||
and a.level + 1 = b.level
|
||||
and a.tilex / 2 = b.tilex
|
||||
and a.tiley / 2 = b.tiley
|
||||
@@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Navigate to the img directory
|
||||
cd img || exit
|
||||
|
||||
# Loop through each folder in the img directory
|
||||
for folder in IMG_*; do
|
||||
# Check if the folder exists and is a directory
|
||||
if [ -d "$folder" ]; then
|
||||
# Navigate into the folder
|
||||
cd "$folder" || continue
|
||||
|
||||
for file in *; do
|
||||
if [[ -f "$file" ]] && ! [[ "$file" =~ ^IMG_[0-9]{4}\.jpeg$ ]]; then
|
||||
rm "$file"
|
||||
fi
|
||||
done
|
||||
|
||||
# Navigate back to the img directory
|
||||
cd ..
|
||||
fi
|
||||
done
|
||||
@@ -0,0 +1,33 @@
|
||||
# def iter_and_append_from_csv(csvfile):
|
||||
# """
|
||||
# Read and iterate over a CSV file.
|
||||
# Args:
|
||||
# csvfile (file): A file object representing the CSV file to read.
|
||||
|
||||
# Yields:
|
||||
# list: A row from the CSV file as a list of strings.
|
||||
# """
|
||||
# csvreader = csv.reader(csvfile)
|
||||
# for row in csvreader:
|
||||
# yield row
|
||||
|
||||
# def _read_csv_data(filename: str, row_indices: List[int], col_indices: List[int]) -> List[List[str]]:
|
||||
# """
|
||||
# Helper function to read and extract specific rows and columns from a CSV file.
|
||||
|
||||
# Args:
|
||||
# filename (str): Path to the CSV file.
|
||||
# row_indices (List[int]): List of row indices to extract.
|
||||
# col_indices (List[int]): List of column indices to extract.
|
||||
|
||||
# Returns:
|
||||
# List[List[str]]: A list of rows, where each row is a list of values at the specified column indices.
|
||||
# """
|
||||
# try:
|
||||
# with open(filename, 'r', encoding='utf-8') as csvfile:
|
||||
# csvfile_data = list(iter_and_append_from_csv(csvfile))
|
||||
# return [[csvfile_data[row][col] for col in col_indices] for row in row_indices]
|
||||
# except FileNotFoundError as e:
|
||||
# print(e)
|
||||
# return []
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Script to test the process_image_analysis function in db.py
|
||||
# This script will run the function and stop on errors.
|
||||
|
||||
# Define the path to the Python script and the image to process
|
||||
# PYTHON_SCRIPT="src/db.py"
|
||||
# Replace with the actual image path
|
||||
IMAGE_PATH="/home/kfl/dev/git/avdb-py-v1/img/TESTIMG/TESTIMG.jpeg"
|
||||
|
||||
# Function to run the test
|
||||
test_process_image_analysis() {
|
||||
echo "Starting test for process_image_analysis..."
|
||||
|
||||
# Run the Python script and capture the output
|
||||
output=$(python3 -c "
|
||||
import sys
|
||||
sys.path.append('src')
|
||||
from db import DBOps, DBConnection
|
||||
|
||||
# Initialize the database connection
|
||||
db_conn = DBConnection('testdb')
|
||||
db_ops = DBOps(db_conn)
|
||||
|
||||
# Run the process_image_analysis function
|
||||
try:
|
||||
# raise(ValueError('testerr'))
|
||||
db_ops.process_image_analysis('$IMAGE_PATH')
|
||||
print('process_image_analysis completed successfully.')
|
||||
except Exception as e:
|
||||
print(f'Error in process_image_analysis: {e}')
|
||||
sys.exit(1)
|
||||
" 2>&1 | tee error_log.txt
|
||||
)
|
||||
# Check the output for errors
|
||||
if [[ "$output" == *"Error"* ]]; then
|
||||
echo "Test failed: $output"
|
||||
exit 1
|
||||
else
|
||||
echo "Test passed: $output"
|
||||
fi
|
||||
}
|
||||
|
||||
# Run the test
|
||||
test_process_image_analysis
|
||||
|
||||
# Exit with the appropriate status
|
||||
# exit $?
|
||||
Reference in New Issue
Block a user