initial commit of new version

tests/TEST.md

@@ -0,0 +1,51 @@
+# Test Plan for `ColorFeaturesPyramid`
+
+## Overview
+This document outlines the plan for testing and documenting the `ColorFeaturesPyramid` class.
+
+## 1. Understanding the `ColorFeaturesPyramid` Class
+- The `ColorFeaturesPyramid` class performs pyramid down-sampling and color feature analysis on images.
+- It relies on the `ColorFeaturesTile` class for tile-based analysis.
+- Key parameters:
+  - `uuid`: Identifier for the image.
+  - `tilew` and `tileh`: Dimensions of the tiles for analysis.
+  - `down_sampling`: Number of down-sampling levels.
+  - `colorspace`: Color space for analysis (e.g., `lab`, `rgb`, `hsv`).
+- The class reads an image, down-samples it, and analyzes the tiles at each level.
+
+## 2. Design the Test Function
+### Test Scope
+Unit tests to verify the functionality of the `ColorFeaturesPyramid` class.
+
+### Test Cases
+1. **Initialization Test**:
+   - Verify that the class initializes correctly with default and custom parameters.
+   - Ensure that the `check_params` function from `utils.py` is used correctly.
+2. **Down-Sampling Test**:
+   - Verify that the down-sampling process works as expected.
+   - Check that the down-sampled images are saved correctly.
+3. **Tile Analysis Test**:
+   - Verify that the tile analysis produces the expected output.
+   - Ensure that the histograms are computed correctly.
+4. **Edge Cases**:
+   - Test with invalid inputs (e.g., non-existent image, invalid color space).
+   - Test with extreme values for parameters (e.g., very large or small tile sizes).
+
+### Testing Framework
+Use `pytest` for writing and running the tests.
+
+## 3. Document the Test Function
+- Write clear docstrings for the test function.
+- Include examples of how to use the `ColorFeaturesPyramid` class.
+- Document the expected behavior and assumptions.
+- Add comments to explain the purpose of each test case.
+
+## 4. Plan the Implementation
+- Create a new test file: `test_color_features_pyramid.py`.
+- Implement the test function and test cases.
+- Ensure the test function is modular and reusable.
+- Use the `check_params` function from `utils.py` to validate parameters.
+
+## 5. Verify the Plan
+- Review the plan to ensure all aspects are covered.
+- Confirm that the test function aligns with the purpose of the `ColorFeaturesPyramid` class.

tests/__init__.py

@@ -0,0 +1,3 @@
+# tests/__init__.py
+
+# This file can be empty or include test-specific initialization

tests/run_tests.py

@@ -0,0 +1,8 @@
+import unittest
+
+if __name__ == '__main__':
+    # Discover and run all tests in the tests directory
+    loader = unittest.TestLoader()
+    suite = loader.discover('tests', pattern='test_*.py')
+    runner = unittest.TextTestRunner()
+    runner.run(suite)

tests/test_alembic_migration.py

@@ -0,0 +1,52 @@
+"""
+Test script to verify the Alembic migration for the avdb_image table.
+"""
+import sqlalchemy
+from sqlalchemy import inspect, text
+
+# Database URL from alembic.ini
+DATABASE_URL = "postgresql+psycopg2://postgres:pgRocks@localhost:5432/testdb"
+
+def test_avdb_image_table_exists():
+    """Test if the avdb_image table exists in the database."""
+    engine = sqlalchemy.create_engine(DATABASE_URL)
+    inspector = inspect(engine)
+    
+    # Print database and table configuration
+    print("Database URL:", DATABASE_URL)
+    print("\nChecking for avdb_image table...")
+    
+    # Check if the table exists
+    assert inspector.has_table("avdb_image"), "avdb_image table does not exist!"
+    print("✓ avdb_image table exists.")
+    
+    # Verify the table structure
+    columns = inspector.get_columns("avdb_image")
+    column_names = [column["name"] for column in columns]
+    
+    print("\nColumns in avdb_image table:")
+    for column in columns:
+        print(f"  - {column['name']}: {column['type']}, Nullable: {column['nullable']}")
+    
+    expected_columns = ["img_id", "img_path", "img_name", "width", "height", "created_at"]
+    for col in expected_columns:
+        assert col in column_names, f"Column '{col}' not found in avdb_image table!"
+    print("\n✓ All expected columns are present in the avdb_image table.")
+    
+    # Verify primary key
+    primary_key = inspector.get_pk_constraint("avdb_image")
+    assert primary_key is not None, "Primary key not found in avdb_image table!"
+    assert primary_key["constrained_columns"] == ["img_id"], "Primary key is not set on img_id!"
+    print("✓ Primary key is correctly set on img_id.")
+    
+    # Verify nullable constraints
+    nullable_columns = [col["name"] for col in columns if col["nullable"]]
+    non_nullable_columns = ["img_path", "img_name", "width", "height"]
+    for col in non_nullable_columns:
+        assert col not in nullable_columns, f"Column '{col}' should not be nullable!"
+    print("✓ Non-nullable constraints are correctly set.")
+    
+    print("\nAll tests passed! The avdb_image table was created successfully.")
+
+if __name__ == "__main__":
+    test_avdb_image_table_exists()

tests/test_color_features_pyramid.py

@@ -0,0 +1,150 @@
+"""
+Test module for the ColorFeaturesPyramid class.
+
+This module contains unit tests to verify the functionality of the ColorFeaturesPyramid class,
+including initialization, down-sampling, and tile analysis.
+"""
+
+import os
+import sys
+import pytest
+import cv2 as cv
+import numpy as np
+
+from color_features_pyramid import ColorFeaturesPyramid
+
+
+class TestColorFeaturesPyramid:
+    """
+    Test suite for the ColorFeaturesPyramid class.
+    """
+
+    @pytest.fixture
+    def setup_image(self, tmp_path):
+        """
+        Fixture to set up a test image for the ColorFeaturesPyramid tests.
+        
+        Args:
+            tmp_path: Pytest fixture providing a temporary directory.
+        
+        Returns:
+            tuple: Path to the test image and the image UUID.
+        """
+        # Create a temporary directory for the test image
+        img_dir = "/home/kfl/dev/git/avdb-py-v1/img"
+        
+        # Create a simple test image
+        img = np.zeros((100, 100, 3), dtype=np.uint16)
+        img[:, :] = (255, 0, 0)  # Red image
+        
+        img_path = img_dir + "/TESTIMG.jpeg"
+        # cv.imwrite(str(img_path), img)
+        
+        return str(img_path), "TESTIMG"
+
+    def test_initialization_default_params(self, setup_image):
+        """
+        Test that the ColorFeaturesPyramid class initializes correctly with default parameters.
+        
+        Args:
+            setup_image: Fixture providing the test image path and UUID.
+        """
+        img_path, uuid = setup_image
+        
+        # Initialize with default parameters
+        pyramid = ColorFeaturesPyramid(uuid=uuid, root_path=os.path.dirname(img_path))
+        
+        # Verify default parameters
+        assert pyramid.uuid == uuid
+        assert pyramid.down_sampling_levels == 9
+        assert pyramid.colorspace == hsv
+
+    def test_initialization_custom_params(self, setup_image):
+        """
+        Test that the ColorFeaturesPyramid class initializes correctly with custom parameters.
+        
+        Args:
+            setup_image: Fixture providing the test image path and UUID.
+        """
+        img_path, uuid = setup_image
+        
+        # Initialize with custom parameters
+        pyramid = ColorFeaturesPyramid(
+            uuid=uuid,
+            root_path=os.path.dirname(img_path),
+            down_sampling=5,
+            colorspace="rgb"
+        )
+        
+        # Verify custom parameters
+        assert pyramid.uuid == uuid
+        assert pyramid.down_sampling_levels == 5
+        assert pyramid.colorspace == "rgb"
+
+    def test_down_sampling(self, setup_image):
+        """
+        Test that the down-sampling process works as expected.
+        
+        Args:
+            setup_image: Fixture providing the test image path and UUID.
+        """
+        img_path, uuid = setup_image
+
+        # Initialize the pyramid
+        pyramid = ColorFeaturesPyramid(uuid=uuid, root_path=os.path.dirname(img_path))
+        
+        # Run the down-sampling and analysis process
+        results = list(pyramid._down_sample_and_analyze())
+        
+        # Verify that results are generated for each level
+        assert len(results) > 0
+        
+        # Verify that down-sampled images are saved
+        for level in range(1, pyramid.down_sampling_levels):
+            # Construct the correct path for the down-sampled image
+            ds_img_filename = f"{uuid}/{uuid}.{pyramid.colorspace}.pyr_{level}.jpeg"
+            ds_img_path = os.path.join(os.path.dirname(img_path), ds_img_filename)
+            assert os.path.exists(ds_img_path), f"Down-sampled image not found: {ds_img_path}"
+
+    def test_tile_analysis(self, setup_image):
+        """
+        Test that the tile analysis produces the expected output.
+        
+        Args:
+            setup_image: Fixture providing the test image path and UUID.
+        """
+        img_path, uuid = setup_image
+        
+        # Initialize the pyramid
+        pyramid = ColorFeaturesPyramid(uuid=uuid, root_path=os.path.dirname(img_path))
+        
+        # Run the down-sampling and analysis process
+        results = list(pyramid._down_sample_and_analyze())
+        
+        # Verify that results contain histograms and metadata
+        for level, (histograms, metadata) in results:
+            assert histograms is not None
+            assert len(metadata) == 4  # (imgw, imgh, tilesw, tilesh)
+
+    def test_edge_case_invalid_image(self):
+        """
+        Test that the class handles invalid image paths gracefully.
+        """
+        # Initialize with a non-existent image
+        with pytest.raises(Exception):
+            pyramid = ColorFeaturesPyramid(uuid="nonexistent_image", root_path="/tmp")
+            list(pyramid._down_sample_and_analyze())
+
+    def test_edge_case_invalid_colorspace(self, setup_image):
+        """
+        Test that the class handles invalid color spaces gracefully.
+        
+        Args:
+            setup_image: Fixture providing the test image path and UUID.
+        """
+        img_path, uuid = setup_image
+        
+        # Initialize with an invalid color space
+        with pytest.raises(Exception):
+            pyramid = ColorFeaturesPyramid(uuid=uuid, root_path=os.path.dirname(img_path), colorspace="invalid")
+            list(pyramid._down_sample_and_analyze())

tests/test_db.py

@@ -0,0 +1,131 @@
+import unittest
+import os
+import numpy as np
+import csv
+import uuid
+from unittest.mock import MagicMock, patch
+import psycopg2
+from psycopg2 import extras
+
+from avdb.db import DBOps
+from avdb.analysis import convert_mmap_to_csv
+
+class TestDBOps(unittest.TestCase):
+    def setUp(self):
+        # Mock the database connection and cursor
+        self.mock_conn = MagicMock()
+        self.mock_cursor = MagicMock()
+        self.mock_conn.get_cursor.return_value = self.mock_cursor
+        self.mock_conn.get_connection.return_value = self.mock_conn
+
+        # Initialize DBOps with the mocked connection
+        self.db_ops = DBOps(self.mock_conn)
+
+    def tearDown(self):
+        # Clean up any temporary files created during tests
+        if os.path.exists("test_histogram.mmap"):
+            os.remove("test_histogram.mmap")
+        if os.path.exists("test_histogram.csv"):
+            os.remove("test_histogram.csv")
+        if os.path.exists("test_tile.mmap"):
+            os.remove("test_tile.mmap")
+
+    def test_convert_mmap_to_csv(self):
+        # Create a temporary .mmap file for testing
+        mmap_file_path = "test_histogram.mmap"
+        csv_file_path = "test_histogram.csv"
+
+        # Create dummy data for the .mmap file
+        dummy_data = np.random.rand(81, 48).astype('float32')
+        with open(mmap_file_path, 'wb') as f:
+            f.write(dummy_data.tobytes())
+
+        # Call the conversion function
+        convert_mmap_to_csv(mmap_file_path, csv_file_path)
+
+        # Verify the CSV file was created and contains the correct data
+        with open(csv_file_path, 'r') as csv_file:
+            reader = csv.reader(csv_file)
+            csv_data = list(reader)
+
+        # Convert the CSV data back to a numpy array for comparison
+        csv_data_array = np.array(csv_data, dtype='float32')
+
+        # Assert that the data matches
+        np.testing.assert_array_equal(csv_data_array, dummy_data)
+
+    def test_insert_tile_data(self):
+        # Create dummy tile data
+        dummy_rows = [
+            (
+                1,  # img_id
+                3,  # level
+                0,  # tilex
+                0,  # tiley
+                100,  # x
+                100,  # y
+                50,  # x0
+                50,  # y0
+                10,  # w
+                10,  # h
+                str(uuid.uuid4()),  # hash
+                [1, 2, 3]  # tile_data
+            ),
+            (
+                1,  # img_id
+                3,  # level
+                1,  # tilex
+                1,  # tiley
+                200,  # x
+                200,  # y
+                100,  # x0
+                100,  # y0
+                10,  # w
+                10,  # h
+                str(uuid.uuid4()),  # hash
+                [4, 5, 6]  # tile_data
+            )
+        ]
+
+        # Mock the execute_batch method
+        with patch('avdb.db.psycopg2.extras.execute_batch') as mock_execute_batch:
+            # Call the insert_tile_data method
+            self.db_ops.insert_tile_data(dummy_rows)
+
+            # Verify that execute_batch was called with the correct query and data
+            mock_execute_batch.assert_called_once_with(
+                self.mock_cursor,
+                """
+                INSERT INTO pyramiddata_level3 (
+                    img_id, level, tilex, tiley, x, y, x0, y0, w, h, hash, tile_data
+                )
+                VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
+                """,
+                dummy_rows
+            )
+        
+    def test_process_and_insert_tile_data_batched(self):
+        # Create a temporary .mmap file for testing
+        mmap_file_path = "test_tile.mmap"
+        dummy_data = np.random.randint(0, 255, size=(100, 3), dtype=np.uint16)
+        with open(mmap_file_path, 'wb') as f:
+            f.write(dummy_data.tobytes())
+
+        # Mock the load_memmapped_tile_data method to return the dummy data
+        self.db_ops.load_memmapped_tile_data = MagicMock(return_value=dummy_data)
+
+        # Mock the execute_batch method
+        with patch('avdb.db.psycopg2.extras.execute_batch') as mock_execute_batch:
+            # Call the process_and_insert_tile_data_batched method
+            self.db_ops.process_and_insert_tile_data_batched(
+                img_id=1,
+                img_dims=(1000, 1000),
+                level=3,
+                tile_file=mmap_file_path
+            )
+
+            # Verify that execute_batch was called
+            mock_execute_batch.assert_called()
+
+if __name__ == '__main__':
+    unittest.main()