Initialize AMO CRM Data Collection Service with FastAPI, SQLite, and Docker support. Added core application files, including API routes, database models, and configuration settings. Implemented AMO CRM API client for data fetching and established Docker setup for containerization. Included example environment files and initial test cases for API endpoints.

2025-09-08 03:30:22 +03:00 · 2025-09-08 03:30:22 +03:00 · e1cf2d1695
commit e1cf2d1695
31 changed files with 6233 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,211 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
 *$py.class
 # C extensions
 *.so
 # Distribution / packaging
 .Python
 build/
 develop-eggs/
 dist/
 downloads/
 eggs/
 .eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
 wheels/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
 *.egg
 MANIFEST
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
 .nox/
 .coverage
 .coverage.*
 .cache
 nosetests.xml
 coverage.xml
 *.cover
 *.py,cover
 .hypothesis/
 .pytest_cache/
 cover/
 # Translations
 *.mo
 *.pot
 # Django stuff:
 *.log
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
 # Flask stuff:
 instance/
 .webassets-cache
 # Scrapy stuff:
 .scrapy
 # Sphinx documentation
 docs/_build/
 # PyBuilder
 .pybuilder/
 target/
 # Jupyter Notebook
 .ipynb_checkpoints
 # IPython
 profile_default/
 ipython_config.py
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:
 # .python-version
 # pipenv
 #   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
 #   However, in case of collaboration, if having platform-specific dependencies or dependencies
 #   having no cross-platform support, pipenv may install dependencies that don't work, or not
 #   install all needed dependencies.
 #Pipfile.lock
 # poetry
 #   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
 #   This is especially recommended for binary packages to ensure reproducibility, and is more
 #   commonly ignored for libraries.
 #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
 #poetry.lock
 # pdm
 #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
 #pdm.lock
 #   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
 #   in version control.
 #   https://pdm.fming.dev/#use-with-ide
 .pdm.toml
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
 __pypackages__/
 # Celery stuff
 celerybeat-schedule
 celerybeat.pid
 # SageMath parsed files
 *.sage.py
 # Environments
 .env
 .venv
 env/
 venv/
 ENV/
 env.bak/
 venv.bak/
 # Spyder project settings
 .spyderproject
 .spyproject
 # Rope project settings
 .ropeproject
 # mkdocs documentation
 /site
 # mypy
 .mypy_cache/
 .dmypy.json
 dmypy.json
 # Pyre type checker
 .pyre/
 # pytype static type analyzer
 .pytype/
 # Cython debug symbols
 cython_debug/
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be added to the global gitignore or merged into this project gitignore.
 #  https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 .idea/
 # Visual Studio Code
 .vscode/
 # Sublime Text
 *.sublime-project
 *.sublime-workspace
 # Database files
 *.db
 *.sqlite
 *.sqlite3
 # Log files
 *.log
 logs/
 # Redis dump files
 dump.rdb
 # Docker
 .dockerignore
 # OS generated files
 .DS_Store
 .DS_Store?
 ._*
 .Spotlight-V100
 .Trashes
 ehthumbs.db
 Thumbs.db
 # Temporary files
 *.tmp
 *.temp
 *.swp
 *.swo
 *~
 # AMO CRM specific
 amocrm_data/
 exports/
 uploads/
 # Local configuration files (keep examples)
 .env.local
 .env.production
 .env.development
 # Google API credentials
 credentials.json
 token.json
--- a/37
+++ b/37
@ -0,0 +1,37 @@
 # Use Python 3.12 slim image
 FROM python:3.12-slim
 # Set working directory
 WORKDIR /app
 # Install system dependencies
 RUN apt-get update && apt-get install -y \
    gcc \
    && rm -rf /var/lib/apt/lists/*
 # Copy pyproject.toml and uv.lock first for better caching
 COPY pyproject.toml uv.lock ./
 # Install uv for faster package management
 RUN pip install uv
 # Install dependencies
 RUN uv sync --frozen
 # Copy application code
 COPY . .
 # Create non-root user
 RUN useradd --create-home --shell /bin/bash app \
    && chown -R app:app /app
 USER app
 # Expose port
 EXPOSE 8000
 # Health check
 HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 \
    CMD python -c "import requests; requests.get('http://localhost:8000/health')" || exit 1
 # Run the application
 CMD ["python", "-m", "uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8000"]
--- a/README.md
+++ b/README.md
@ -0,0 +1,277 @@
 # AMO CRM Data Collection Service
 A FastAPI service that collects data from AMO CRM API and stores it in SQLite database for further export to Google Sheets.
 ## Features
 - **Data Collection**: Collects Deals, Contacts, Companies, Pipelines, Users, and Events from AMO CRM
 - **HAL+JSON Support**: Handles AMO CRM's HAL+JSON response format
 - **Custom Fields**: Extracts and stores custom fields with proper typing
 - **Relationships**: Maintains proper relationships between entities
 - **Google Sheets Export**: Unified configuration for exporting all entities to Google Sheets
 - **Background Processing**: Async job processing for exports
 - **SQLite Database**: Optimized storage with proper indexing
 ## Project Structure
 ```
 amo-server/
 ├── adapters/           # Database and external service adapters
 │   └── sqlite/         # SQLite database models and connection
 ├── routers/            # FastAPI route handlers
 ├── servers/            # Business logic (to be implemented)
 ├── utils/              # Utilities and configuration
 ├── workers/            # Background workers (to be implemented)
 ├── tests/              # Test files and fixtures
 │   └── fixtures/       # AMO CRM response examples
 └── docs/               # Documentation
 ```
 ## Setup
 ### Prerequisites
 - Python 3.12+
 - [uv](https://docs.astral.sh/uv/) package manager
 ### Installation
 1. Clone the repository:
 ```bash
 git clone <repository-url>
 cd amo-server
 ```
 2. Install dependencies using uv:
 ```bash
 uv sync
 ```
 3. Create environment file:
 ```bash
 cp env.example .env
 ```
 4. Configure your environment variables in `.env`:
 ```env
 # Database
 DATABASE_URL=sqlite:///./amo_data.db
 # AMO CRM API
 AMO_CRM_DOMAIN=wecheap.amocrm.ru
 AMO_CRM_ACCESS_TOKEN=your-longterm-access-token
 # Google Sheets API
 GOOGLE_SERVICE_ACCOUNT_FILE=path/to/service-account.json
 # Redis (for Celery)
 REDIS_URL=redis://localhost:6379/0
 # API Settings
 API_V1_STR=/api/v1
 # Logging
 LOG_LEVEL=INFO
 ```
 ### Running the Service
 1. Start the development server:
 ```bash
 uv run uvicorn app:app --reload
 ```
 2. Open your browser to http://localhost:8000 to see the API
 3. Visit http://localhost:8000/docs for the interactive API documentation
 ### Fetching Real AMO CRM Data
 To fetch real responses from your AMO CRM for testing:
 1. Set your access token in `.env`:
 ```env
 AMO_CRM_ACCESS_TOKEN=your-longterm-token
 ```
 2. Run the fetch script:
 ```bash
 uv run python scripts/fetch_amocrm_data.py
 ```
 This will:
 - Fetch real data from wecheap.amocrm.ru
 - Save individual JSON files to `tests/fixtures/real_responses/`
 - Generate Python fixtures at `tests/fixtures/real_amocrm_responses.py`
 - Show a summary of fetched data
 3. Test the API connection:
 ```bash
 curl http://localhost:8000/api/v1/amocrm/info
 curl http://localhost:8000/api/v1/amocrm/fetch/users?limit=5
 ```
 ## API Endpoints
 ### Entity Management
 - `GET /api/v1/entities` - List all entities with statistics
 - `GET /api/v1/entities/{entity_type}/fields` - List fields for specific entity
 ### AMO CRM Integration
 - `GET /api/v1/amocrm/info` - Get AMO CRM connection info
 - `GET /api/v1/amocrm/fetch/{entity_type}` - Fetch data directly from AMO CRM
 - `GET /api/v1/amocrm/fetch/custom_fields/{entity_type}` - Fetch custom fields metadata
 - `GET /api/v1/amocrm/fetch/all` - Fetch all data from AMO CRM (for testing)
 - `POST /api/v1/amocrm/sync/{entity_type}` - Fetch and sync data from AMO CRM
 ### Export Management
 - `POST /api/v1/export/configure` - Create unified export configuration
 - `GET /api/v1/export/configurations` - List export configurations
 - `POST /api/v1/export/start` - Start export job
 - `GET /api/v1/export/status/{job_id}` - Get export job status
 - `GET /api/v1/export/jobs` - List all export jobs
 ### Data Ingestion (Worker API)
 - `POST /api/v1/data/users` - Import users data
 - `POST /api/v1/data/pipelines` - Import pipelines data
 - `POST /api/v1/data/companies` - Import companies data
 - `POST /api/v1/data/contacts` - Import contacts data
 - `POST /api/v1/data/deals` - Import deals data
 - `POST /api/v1/data/events` - Import events data
 ## Database Schema
 The service uses SQLite with the following main tables:
 - `amo_users` - CRM users
 - `amo_pipelines` - Sales pipelines
 - `amo_pipeline_stages` - Pipeline stages
 - `amo_companies` - Companies
 - `amo_contacts` - Contacts
 - `amo_deals` - Deals/Leads
 - `amo_events` - Activity events
 - `amo_custom_fields` - Universal custom fields storage
 - `export_configuration` - Export configurations
 - `export_entity_mappings` - Entity-specific export mappings
 - `export_jobs` - Export job tracking
 ## AMO CRM Response Examples
 The project includes real AMO CRM API response examples in `tests/fixtures/amocrm_responses.py` for:
 - **Users** (`USERS_RESPONSE`)
 - **Pipelines** (`PIPELINES_RESPONSE`)
 - **Companies** (`COMPANIES_RESPONSE`)
 - **Contacts** (`CONTACTS_RESPONSE`)
 - **Deals** (`DEALS_RESPONSE`)
 - **Events** (`EVENTS_RESPONSE`)
 - **Custom Fields Metadata** (`CUSTOM_FIELDS_RESPONSE`)
 These examples demonstrate:
 - HAL+JSON format structure
 - Custom fields with different types (text, select, numeric, date)
 - Embedded relationships between entities
 - Proper date formats (Unix timestamps)
 - Event types filtering (incoming_call, outgoing_call, lead_status_changed)
 ## Testing
 Run tests using pytest:
 ```bash
 uv run pytest
 ```
 Run tests with coverage:
 ```bash
 uv run pytest --cov=. --cov-report=html
 ```
 ## Export Configuration Example
 Create a unified export configuration for all entities:
 ```json
 {
  "name": "Q1 2024 AMO CRM Export",
  "sheet_id": "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms",
  "date_range_start": "2024-01-01T00:00:00Z",
  "date_range_end": "2024-03-31T23:59:59Z",
  "entity_mappings": {
    "deals": {
      "sheet_name": "Deals",
      "is_enabled": true,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "price", "column": "B", "order": 2},
        {"field_name": "created_at", "column": "C", "order": 3}
      ]
    },
    "contacts": {
      "sheet_name": "Contacts",
      "is_enabled": true,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "email", "column": "B", "order": 2}
      ]
    }
  }
 }
 ```
 ## Custom Fields Handling
 The service automatically extracts custom fields from AMO CRM responses:
 - **Field Types**: text, numeric, select, multiselect, date, checkbox, textarea, url
 - **Type Detection**: Automatically detects and converts field values
 - **Storage**: Stores in universal `amo_custom_fields` table with proper typing
 - **Relationships**: Links custom fields to their parent entities
 ## Date Validation
 All dates are validated to be between 2017-2026 (Unix timestamps: 1483228800 - 1767225600).
 ## Development
 ### Code Style
 The project uses:
 - **Black** for code formatting
 - **isort** for import sorting
 - **flake8** for linting
 - **mypy** for type checking
 Run code formatting:
 ```bash
 uv run black .
 uv run isort .
 ```
 ### Project Conventions
 Following the established patterns:
 - **Adapters**: Handle external integrations (database, APIs)
 - **Servers**: Contain business logic
 - **Routers**: Define API routes only
 - **Workers**: Background processing
 - **Utils**: Shared utilities and configuration
 ## Next Steps
 1. **Implement Background Workers**: Set up Celery workers for data collection
 2. **Add Google Sheets Integration**: Implement actual Google Sheets export
 3. **Add AMO CRM Client**: Create HTTP client for AMO CRM API
 4. **Add Authentication**: Implement API authentication
 5. **Add Monitoring**: Set up logging and metrics
 6. **Add Rate Limiting**: Implement API rate limiting
 7. **Add Caching**: Use Redis for caching frequently accessed data
 ## License
 MIT License
--- a/adapters/init.py
+++ b/adapters/init.py
@ -0,0 +1 @@
 # Adapters package
--- a/adapters/amocrm_client.py
+++ b/adapters/amocrm_client.py
@ -0,0 +1,124 @@
 """
 AMO CRM API Client
 Simple client for fetching data from AMO CRM API
 """
 import httpx
 import asyncio
 from typing import Dict, Any, List, Optional
 from utils.config import settings
 class AmoCRMClient:
    def __init__(self, domain: str = None, access_token: str = None):
        self.domain = domain or settings.AMO_CRM_DOMAIN
        self.access_token = access_token or settings.AMO_CRM_ACCESS_TOKEN
        self.base_url = f"https://{self.domain}/api/v4"
        self.headers = {
            "Authorization": f"Bearer {self.access_token}",
            "Content-Type": "application/json",
        }
    async def _make_request(self, endpoint: str, params: Dict[str, Any] = None) -> Dict[str, Any]:
        """Make HTTP request to AMO CRM API"""
        async with httpx.AsyncClient() as client:
            url = f"{self.base_url}/{endpoint}"
            response = await client.get(url, headers=self.headers, params=params)
            response.raise_for_status()
            return response.json()
    async def get_users(self, limit: int = 250) -> Dict[str, Any]:
        """Get users (пользователи)"""
        return await self._make_request("users", {"limit": limit})
    async def get_pipelines(self) -> Dict[str, Any]:
        """Get pipelines (воронки)"""
        return await self._make_request("leads/pipelines")
    async def get_companies(self, limit: int = 250, page: int = 1) -> Dict[str, Any]:
        """Get companies (компании)"""
        params = {"limit": limit, "page": page, "with": "custom_fields_values"}
        return await self._make_request("companies", params)
    async def get_contacts(self, limit: int = 250, page: int = 1) -> Dict[str, Any]:
        """Get contacts (контакты)"""
        params = {"limit": limit, "page": page, "with": "custom_fields_values"}
        return await self._make_request("contacts", params)
    async def get_deals(self, limit: int = 250, page: int = 1) -> Dict[str, Any]:
        """Get deals/leads (сделки)"""
        params = {"limit": limit, "page": page, "with": "contacts,companies,custom_fields_values"}
        return await self._make_request("leads", params)
    async def get_events(self, limit: int = 250, page: int = 1, event_types: List[str] = None) -> Dict[str, Any]:
        """Get events (события)"""
        params = {"limit": limit, "page": page}
        # Filter by event types
        if event_types:
            params["filter[type]"] = event_types
        else:
            # Default to our supported event types
            params["filter[type]"] = ["incoming_call", "outgoing_call", "lead_status_changed"]
        return await self._make_request("events", params)
    async def get_custom_fields(self, entity_type: str = "leads") -> Dict[str, Any]:
        """Get custom fields metadata for entity type"""
        return await self._make_request(f"{entity_type}/custom_fields")
    async def fetch_all_data(self) -> Dict[str, Any]:
        """Fetch all data from AMO CRM for testing"""
        print("Fetching AMO CRM data...")
        results = {}
        try:
            print("- Fetching users...")
            results["users"] = await self.get_users()
            print(f"  Found {len(results['users'].get('_embedded', {}).get('users', []))} users")
            print("- Fetching pipelines...")
            results["pipelines"] = await self.get_pipelines()
            print(f"  Found {len(results['pipelines'].get('_embedded', {}).get('pipelines', []))} pipelines")
            print("- Fetching companies...")
            results["companies"] = await self.get_companies(limit=10)  # Limit for testing
            print(f"  Found {len(results['companies'].get('_embedded', {}).get('companies', []))} companies")
            print("- Fetching contacts...")
            results["contacts"] = await self.get_contacts(limit=10)  # Limit for testing
            print(f"  Found {len(results['contacts'].get('_embedded', {}).get('contacts', []))} contacts")
            print("- Fetching deals...")
            results["deals"] = await self.get_deals(limit=10)  # Limit for testing
            print(f"  Found {len(results['deals'].get('_embedded', {}).get('leads', []))} deals")
            print("- Fetching events...")
            results["events"] = await self.get_events(limit=10)  # Limit for testing
            print(f"  Found {len(results['events'].get('_embedded', {}).get('events', []))} events")
            print("- Fetching custom fields...")
            results["custom_fields_deals"] = await self.get_custom_fields("leads")
            results["custom_fields_contacts"] = await self.get_custom_fields("contacts")
            results["custom_fields_companies"] = await self.get_custom_fields("companies")
            print("✅ All data fetched successfully!")
        except Exception as e:
            print(f"❌ Error fetching data: {e}")
            raise
        return results
 # Convenience function for testing
 async def fetch_amocrm_data():
    """Fetch AMO CRM data for testing"""
    client = AmoCRMClient()
    return await client.fetch_all_data()
 if __name__ == "__main__":
    # Test the client
    asyncio.run(fetch_amocrm_data())
--- a/adapters/sqlite/init.py
+++ b/adapters/sqlite/init.py
@ -0,0 +1 @@
 # SQLite adapter package
--- a/adapters/sqlite/database.py
+++ b/adapters/sqlite/database.py
@ -0,0 +1,41 @@
 from sqlalchemy import create_engine
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.pool import StaticPool
 from utils.config import settings
 # Create SQLite engine with proper configuration
 engine = create_engine(
    settings.DATABASE_URL,
    connect_args={
        "check_same_thread": False,  # Allow multiple threads for SQLite
        "timeout": 20,  # Set timeout for database operations
    },
    poolclass=StaticPool,
    echo=settings.LOG_LEVEL == "DEBUG",  # Log SQL queries in debug mode
 )
 # Create session factory
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 # Create base class for models
 Base = declarative_base()
 def get_db():
    """Dependency to get database session"""
    db = SessionLocal()
    try:
        yield db
    finally:
        db.close()
 async def init_db():
    """Initialize database tables"""
    # Import all models to ensure they're registered
    from adapters.sqlite import models  # noqa: F401
    # Create all tables
    Base.metadata.create_all(bind=engine)
--- a/adapters/sqlite/migrations/env.py
+++ b/adapters/sqlite/migrations/env.py
@ -0,0 +1,84 @@
 from logging.config import fileConfig
 from sqlalchemy import engine_from_config, pool
 from alembic import context
 import sys
 import os
 # Add the project root to Python path
 sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
 from adapters.sqlite.database import Base
 from adapters.sqlite.models import *  # Import all models
 from utils.config import settings
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 # Interpret the config file for Python logging.
 if config.config_file_name is not None:
    fileConfig(config.config_file_name)
 # Set the SQLAlchemy URL from settings
 config.set_main_option("sqlalchemy.url", settings.DATABASE_URL)
 # add your model's MetaData object here
 # for 'autogenerate' support
 target_metadata = Base.metadata
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
 # ... etc.
 def run_migrations_offline() -> None:
    """Run migrations in 'offline' mode.
    This configures the context with just a URL
    and not an Engine, though an Engine is acceptable
    here as well.  By skipping the Engine creation
    we don't even need a DBAPI to be available.
    Calls to context.execute() here emit the given string to the
    script output.
    """
    url = config.get_main_option("sqlalchemy.url")
    context.configure(
        url=url,
        target_metadata=target_metadata,
        literal_binds=True,
        dialect_opts={"paramstyle": "named"},
    )
    with context.begin_transaction():
        context.run_migrations()
 def run_migrations_online() -> None:
    """Run migrations in 'online' mode.
    In this scenario we need to create an Engine
    and associate a connection with the context.
    """
    connectable = engine_from_config(
        config.get_section(config.config_ini_section, {}),
        prefix="sqlalchemy.",
        poolclass=pool.NullPool,
    )
    with connectable.connect() as connection:
        context.configure(
            connection=connection, target_metadata=target_metadata
        )
        with context.begin_transaction():
            context.run_migrations()
 if context.is_offline_mode():
    run_migrations_offline()
 else:
    run_migrations_online()
--- a/adapters/sqlite/migrations/script.py.mako
+++ b/adapters/sqlite/migrations/script.py.mako
@ -0,0 +1,24 @@
 """${message}
 Revision ID: ${up_revision}
 Revises: ${down_revision | comma,n}
 Create Date: ${create_date}
 """
 from alembic import op
 import sqlalchemy as sa
 ${imports if imports else ""}
 # revision identifiers, used by Alembic.
 revision = ${repr(up_revision)}
 down_revision = ${repr(down_revision)}
 branch_labels = ${repr(branch_labels)}
 depends_on = ${repr(depends_on)}
 def upgrade() -> None:
    ${upgrades if upgrades else "pass"}
 def downgrade() -> None:
    ${downgrades if downgrades else "pass"}
--- a/adapters/sqlite/models.py
+++ b/adapters/sqlite/models.py
@ -0,0 +1,281 @@
 from sqlalchemy import (
    Column, Integer, String, Boolean, Text, ForeignKey, JSON,
    Table, Index, CheckConstraint
 )
 from sqlalchemy.orm import relationship
 from sqlalchemy.ext.declarative import declarative_base
 from .database import Base
 # Association tables for many-to-many relationships
 deal_contacts = Table(
    'amo_deal_contacts',
    Base.metadata,
    Column('deal_id', Integer, ForeignKey('amo_deals.id'), primary_key=True),
    Column('contact_id', Integer, ForeignKey('amo_contacts.id'), primary_key=True),
    Column('is_main', Boolean, default=False),
 )
 deal_companies = Table(
    'amo_deal_companies',
    Base.metadata,
    Column('deal_id', Integer, ForeignKey('amo_deals.id'), primary_key=True),
    Column('company_id', Integer, ForeignKey('amo_companies.id'), primary_key=True),
    Column('is_main', Boolean, default=False),
 )
 contact_companies = Table(
    'amo_contact_companies',
    Base.metadata,
    Column('contact_id', Integer, ForeignKey('amo_contacts.id'), primary_key=True),
    Column('company_id', Integer, ForeignKey('amo_companies.id'), primary_key=True),
    Column('is_main', Boolean, default=False),
 )
 class User(Base):
    __tablename__ = 'amo_users'
    id = Column(Integer, primary_key=True)
    name = Column(String(255), nullable=False)
    email = Column(String(255))
    is_active = Column(Boolean, default=True)
    created_at = Column(Integer)
    updated_at = Column(Integer)
    raw_data = Column(JSON)
    # Relationships
    created_deals = relationship("Deal", foreign_keys="Deal.created_by", back_populates="creator")
    updated_deals = relationship("Deal", foreign_keys="Deal.updated_by", back_populates="updater")
    responsible_deals = relationship("Deal", foreign_keys="Deal.responsible_user_id", back_populates="responsible_user")
    created_contacts = relationship("Contact", foreign_keys="Contact.created_by", back_populates="creator")
    updated_contacts = relationship("Contact", foreign_keys="Contact.updated_by", back_populates="updater")
    responsible_contacts = relationship("Contact", foreign_keys="Contact.responsible_user_id", back_populates="responsible_user")
    created_companies = relationship("Company", foreign_keys="Company.created_by", back_populates="creator")
    updated_companies = relationship("Company", foreign_keys="Company.updated_by", back_populates="updater")
    responsible_companies = relationship("Company", foreign_keys="Company.responsible_user_id", back_populates="responsible_user")
    events = relationship("Event", back_populates="created_by_user")
 class Pipeline(Base):
    __tablename__ = 'amo_pipelines'
    id = Column(Integer, primary_key=True)
    name = Column(String(255), nullable=False)
    sort = Column(Integer)
    is_main = Column(Boolean, default=False)
    is_unsorted = Column(Boolean, default=False)
    is_archive = Column(Boolean, default=False)
    account_id = Column(Integer)
    created_at = Column(Integer)
    updated_at = Column(Integer)
    raw_data = Column(JSON)
    # Relationships
    stages = relationship("PipelineStage", back_populates="pipeline")
    deals = relationship("Deal", back_populates="pipeline")
 class PipelineStage(Base):
    __tablename__ = 'amo_pipeline_stages'
    id = Column(Integer, primary_key=True)
    pipeline_id = Column(Integer, ForeignKey('amo_pipelines.id'), nullable=False)
    name = Column(String(255), nullable=False)
    sort = Column(Integer)
    is_editable = Column(Boolean, default=True)
    color = Column(String(7))  # Hex color code
    created_at = Column(Integer)
    updated_at = Column(Integer)
    raw_data = Column(JSON)
    # Relationships
    pipeline = relationship("Pipeline", back_populates="stages")
    deals = relationship("Deal", back_populates="status")
 class Company(Base):
    __tablename__ = 'amo_companies'
    id = Column(Integer, primary_key=True)
    name = Column(String(255), nullable=False)
    responsible_user_id = Column(Integer, ForeignKey('amo_users.id'))
    group_id = Column(Integer)
    created_by = Column(Integer, ForeignKey('amo_users.id'))
    updated_by = Column(Integer, ForeignKey('amo_users.id'))
    created_at = Column(Integer)
    updated_at = Column(Integer)
    closest_task_at = Column(Integer)
    is_deleted = Column(Boolean, default=False)
    raw_data = Column(JSON)
    # Relationships
    responsible_user = relationship("User", foreign_keys=[responsible_user_id], back_populates="responsible_companies")
    creator = relationship("User", foreign_keys=[created_by], back_populates="created_companies")
    updater = relationship("User", foreign_keys=[updated_by], back_populates="updated_companies")
    # Many-to-many relationships
    deals = relationship("Deal", secondary=deal_companies, back_populates="companies")
    contacts = relationship("Contact", secondary=contact_companies, back_populates="companies")
 class Contact(Base):
    __tablename__ = 'amo_contacts'
    id = Column(Integer, primary_key=True)
    name = Column(String(255), nullable=False)
    first_name = Column(String(255))
    last_name = Column(String(255))
    responsible_user_id = Column(Integer, ForeignKey('amo_users.id'))
    group_id = Column(Integer)
    created_by = Column(Integer, ForeignKey('amo_users.id'))
    updated_by = Column(Integer, ForeignKey('amo_users.id'))
    created_at = Column(Integer)
    updated_at = Column(Integer)
    closest_task_at = Column(Integer)
    is_deleted = Column(Boolean, default=False)
    raw_data = Column(JSON)
    # Relationships
    responsible_user = relationship("User", foreign_keys=[responsible_user_id], back_populates="responsible_contacts")
    creator = relationship("User", foreign_keys=[created_by], back_populates="created_contacts")
    updater = relationship("User", foreign_keys=[updated_by], back_populates="updated_contacts")
    # Many-to-many relationships
    deals = relationship("Deal", secondary=deal_contacts, back_populates="contacts")
    companies = relationship("Company", secondary=contact_companies, back_populates="contacts")
 class Deal(Base):
    __tablename__ = 'amo_deals'
    id = Column(Integer, primary_key=True)
    name = Column(String(255), nullable=False)
    price = Column(Integer, default=0)
    responsible_user_id = Column(Integer, ForeignKey('amo_users.id'))
    group_id = Column(Integer)
    status_id = Column(Integer, ForeignKey('amo_pipeline_stages.id'))
    pipeline_id = Column(Integer, ForeignKey('amo_pipelines.id'))
    loss_reason_id = Column(Integer)
    created_by = Column(Integer, ForeignKey('amo_users.id'))
    updated_by = Column(Integer, ForeignKey('amo_users.id'))
    closed_at = Column(Integer)
    created_at = Column(Integer)
    updated_at = Column(Integer)
    closest_task_at = Column(Integer)
    is_deleted = Column(Boolean, default=False)
    raw_data = Column(JSON)
    # Relationships
    responsible_user = relationship("User", foreign_keys=[responsible_user_id], back_populates="responsible_deals")
    creator = relationship("User", foreign_keys=[created_by], back_populates="created_deals")
    updater = relationship("User", foreign_keys=[updated_by], back_populates="updated_deals")
    status = relationship("PipelineStage", back_populates="deals")
    pipeline = relationship("Pipeline", back_populates="deals")
    # Many-to-many relationships
    contacts = relationship("Contact", secondary=deal_contacts, back_populates="deals")
    companies = relationship("Company", secondary=deal_companies, back_populates="deals")
 class Event(Base):
    __tablename__ = 'amo_events'
    __table_args__ = (
        CheckConstraint(
            "type IN ('incoming_call', 'outgoing_call', 'lead_status_changed')",
            name='check_event_type'
        ),
    )
    id = Column(Integer, primary_key=True)
    type = Column(String(50), nullable=False)
    entity_id = Column(Integer)
    entity_type = Column(String(50))
    created_by = Column(Integer, ForeignKey('amo_users.id'))
    created_at = Column(Integer)
    value_after = Column(JSON)
    value_before = Column(JSON)
    account_id = Column(Integer)
    raw_data = Column(JSON)
    # Relationships
    created_by_user = relationship("User", back_populates="events")
 class CustomField(Base):
    __tablename__ = 'amo_custom_fields'
    id = Column(Integer, primary_key=True, autoincrement=True)
    entity_type = Column(String(50), nullable=False)  # 'deals', 'contacts', 'companies'
    entity_id = Column(Integer, nullable=False)
    field_id = Column(Integer, nullable=False)
    field_name = Column(String(255), nullable=False)
    field_type = Column(String(50), nullable=False)  # 'text', 'numeric', 'checkbox', 'select', etc.
    field_value = Column(Text)
    field_value_numeric = Column(Integer)
    field_value_date = Column(Integer)
    is_custom = Column(Boolean, default=True)
    created_at = Column(Integer)
    updated_at = Column(Integer)
    # Indexes
    __table_args__ = (
        Index('idx_custom_fields_entity', 'entity_type', 'entity_id'),
        Index('idx_custom_fields_field', 'field_id'),
        Index('idx_custom_fields_name', 'field_name'),
    )
 # Export configuration models
 class ExportConfiguration(Base):
    __tablename__ = 'export_configuration'
    id = Column(Integer, primary_key=True, autoincrement=True)
    name = Column(String(255), nullable=False)
    sheet_id = Column(String(255), nullable=False)
    date_range_start = Column(Integer)
    date_range_end = Column(Integer)
    is_active = Column(Boolean, default=True)
    created_at = Column(Integer)
    updated_at = Column(Integer)
    # Relationships
    entity_mappings = relationship("ExportEntityMapping", back_populates="configuration")
    export_jobs = relationship("ExportJob", back_populates="configuration")
 class ExportEntityMapping(Base):
    __tablename__ = 'export_entity_mappings'
    id = Column(Integer, primary_key=True, autoincrement=True)
    configuration_id = Column(Integer, ForeignKey('export_configuration.id'), nullable=False)
    entity_type = Column(String(50), nullable=False)
    sheet_name = Column(String(255), nullable=False)
    field_mapping = Column(JSON, nullable=False)
    is_enabled = Column(Boolean, default=True)
    created_at = Column(Integer)
    updated_at = Column(Integer)
    # Relationships
    configuration = relationship("ExportConfiguration", back_populates="entity_mappings")
 class ExportJob(Base):
    __tablename__ = 'export_jobs'
    id = Column(Integer, primary_key=True, autoincrement=True)
    configuration_id = Column(Integer, ForeignKey('export_configuration.id'), nullable=False)
    status = Column(String(50), nullable=False, default='pending')
    records_processed = Column(Integer, default=0)
    total_records = Column(Integer, default=0)
    error_message = Column(Text)
    started_at = Column(Integer)
    completed_at = Column(Integer)
    created_at = Column(Integer)
    # Relationships
    configuration = relationship("ExportConfiguration", back_populates="export_jobs")
--- a/alembic.ini
+++ b/alembic.ini
@ -0,0 +1,109 @@
 # A generic, single database configuration.
 [alembic]
 # path to migration scripts
 script_location = adapters/sqlite/migrations
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
 # file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.
 prepend_sys_path = .
 # timezone to use when rendering the date within the migration file
 # as well as the filename.
 # If specified, requires the python-dateutil library that can be
 # installed by adding `alembic[tz]` to the pip requirements
 # string value is passed to dateutil.tz.gettz()
 # leave blank for localtime
 # timezone =
 # max length of characters to apply to the
 # "slug" field
 # truncate_slug_length = 40
 # set to 'true' to run the environment during
 # the 'revision' command, regardless of autogenerate
 # revision_environment = false
 # set to 'true' to allow .pyc and .pyo files without
 # a source .py file to be detected as revisions in the
 # versions/ directory
 # sourceless = false
 # version path separator; As mentioned above, this is the character used to split
 # version_locations. The default within new alembic.ini files is "os", which uses
 # os.pathsep. If this key is omitted entirely, it falls back to the legacy
 # behavior of splitting on spaces and/or commas.
 # Valid values for version_path_separator are:
 #
 # version_path_separator = :
 # version_path_separator = ;
 # version_path_separator = space
 version_path_separator = os
 # set to 'true' to search source files recursively
 # in each "version_locations" directory
 # new in Alembic version 1.10
 # recursive_version_locations = false
 # the output encoding used when revision files
 # are written from script.py.mako
 # output_encoding = utf-8
 sqlalchemy.url = sqlite:///./amo_data.db
 [post_write_hooks]
 # post_write_hooks defines scripts or Python functions that are run
 # on newly generated revision scripts.  See the documentation for further
 # detail and examples
 # format using "black" - use the console_scripts runner, against the "black" entrypoint
 # hooks = black
 # black.type = console_scripts
 # black.entrypoint = black
 # black.options = -l 79 REVISION_SCRIPT_FILENAME
 # lint with attempts to fix using "ruff" - use the exec runner, execute a binary
 # hooks = ruff
 # ruff.type = exec
 # ruff.executable = %(here)s/.venv/bin/ruff
 # ruff.options = --fix REVISION_SCRIPT_FILENAME
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic
 [handlers]
 keys = console
 [formatters]
 keys = generic
 [logger_root]
 level = WARN
 handlers = console
 qualname =
 [logger_sqlalchemy]
 level = WARN
 handlers =
 qualname = sqlalchemy.engine
 [logger_alembic]
 level = INFO
 handlers =
 qualname = alembic
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = NOTSET
 formatter = generic
 [formatter_generic]
 format = %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %H:%M:%S
--- a/app.py
+++ b/app.py
@ -0,0 +1,60 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from contextlib import asynccontextmanager
 import uvicorn
 from adapters.sqlite.database import init_db
 from routers import entities, export, data, amocrm
 from utils.config import settings
@asynccontextmanager
 async def lifespan(app: FastAPI):
    # Startup
    await init_db()
    yield
    # Shutdown
    pass
 app = FastAPI(
    title="AMO CRM Data Collection Service",
    description="Service for collecting and exporting AMO CRM data to Google Sheets",
    version="0.1.0",
    lifespan=lifespan,
 )
 # CORS middleware
 app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],  # Configure appropriately for production
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
 )
 # Include routers
 app.include_router(entities.router, prefix=f"{settings.API_V1_STR}/entities", tags=["entities"])
 app.include_router(export.router, prefix=f"{settings.API_V1_STR}/export", tags=["export"])
 app.include_router(data.router, prefix=f"{settings.API_V1_STR}/data", tags=["data"])
 app.include_router(amocrm.router, prefix=f"{settings.API_V1_STR}/amocrm", tags=["amocrm"])
@app.get("/")
 async def root():
    return {"message": "AMO CRM Data Collection Service", "version": "0.1.0"}
@app.get("/health")
 async def health_check():
    return {"status": "healthy"}
 if __name__ == "__main__":
    uvicorn.run(
        "app:app",
        host="0.0.0.0",
        port=8000,
        reload=True,
        log_level=settings.LOG_LEVEL.lower(),
    )
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,83 @@
 version: '3.8'
 services:
  app:
    build: .
    ports:
      - "8000:8000"
    environment:
      - DATABASE_URL=sqlite:///./data/amo_data.db
      - REDIS_URL=redis://redis:6379/0
      - AMO_CRM_DOMAIN=${AMO_CRM_DOMAIN}
      - AMO_CRM_ACCESS_TOKEN=${AMO_CRM_ACCESS_TOKEN}
      - GOOGLE_SERVICE_ACCOUNT_FILE=${GOOGLE_SERVICE_ACCOUNT_FILE}
      - GOOGLE_SCOPES=${GOOGLE_SCOPES}
      - API_V1_STR=/api/v1
      - LOG_LEVEL=INFO
    volumes:
      - ./data:/app/data
      - ./credentials:/app/credentials:ro
    depends_on:
      - redis
    restart: unless-stopped
    networks:
      - amo-network
  redis:
    image: redis:7-alpine
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
    restart: unless-stopped
    networks:
      - amo-network
  celery-worker:
    build: .
    command: python -m celery -A workers.celery_app worker --loglevel=info
    environment:
      - DATABASE_URL=sqlite:///./data/amo_data.db
      - REDIS_URL=redis://redis:6379/0
      - AMO_CRM_DOMAIN=${AMO_CRM_DOMAIN}
      - AMO_CRM_ACCESS_TOKEN=${AMO_CRM_ACCESS_TOKEN}
      - GOOGLE_SERVICE_ACCOUNT_FILE=${GOOGLE_SERVICE_ACCOUNT_FILE}
      - GOOGLE_SCOPES=${GOOGLE_SCOPES}
      - API_V1_STR=/api/v1
      - LOG_LEVEL=INFO
    volumes:
      - ./data:/app/data
      - ./credentials:/app/credentials:ro
    depends_on:
      - redis
    restart: unless-stopped
    networks:
      - amo-network
  celery-beat:
    build: .
    command: python -m celery -A workers.celery_app beat --loglevel=info
    environment:
      - DATABASE_URL=sqlite:///./data/amo_data.db
      - REDIS_URL=redis://redis:6379/0
      - AMO_CRM_DOMAIN=${AMO_CRM_DOMAIN}
      - AMO_CRM_ACCESS_TOKEN=${AMO_CRM_ACCESS_TOKEN}
      - GOOGLE_SERVICE_ACCOUNT_FILE=${GOOGLE_SERVICE_ACCOUNT_FILE}
      - GOOGLE_SCOPES=${GOOGLE_SCOPES}
      - API_V1_STR=/api/v1
      - LOG_LEVEL=INFO
    volumes:
      - ./data:/app/data
      - ./credentials:/app/credentials:ro
    depends_on:
      - redis
    restart: unless-stopped
    networks:
      - amo-network
 volumes:
  redis_data:
 networks:
  amo-network:
    driver: bridge
--- a/docs/amocrm-service-design.md
+++ b/docs/amocrm-service-design.md
@ -0,0 +1,803 @@
 # AMO CRM Data Collection Service - Design Document
 ## Overview
 This service collects data from AMO CRM API (wecheap.amocrm.ru) and stores it in SQLite database for further export to Google Sheets. The service handles AMO CRM's HAL+JSON format and maintains proper relationships between entities.
 **Key Features:**
 - Simple API without authentication
 - Direct AMO CRM integration with long-term tokens
 - Real-time data fetching and testing
 - Unified export configuration for all entities
 - Background job processing for exports
 ## Architecture
 ```
 ┌─────────────────┐    ┌─────────────────┐    ┌─────────────────┐
 │   AMO CRM API   │    │   Our Service   │    │  Google Sheets  │
 │                 │    │                 │    │                 │
 │ HAL+JSON Format │◄──►│ SQLite Database │───►│   Export API    │
 │                 │    │                 │    │                 │
 └─────────────────┘    └─────────────────┘    └─────────────────┘
 ```
 ### Components
 - **Adapters**: Database connections, AMO CRM API client, Google Sheets client
 - **Routers**: API endpoints (no authentication required)
 - **Servers**: Business logic for data processing and export
 - **Workers**: Background data collection and processing
 - **Scripts**: Utilities for fetching real AMO CRM data
 ## Data Entities
 ### 1. Deals (Сделки)
 - Primary entity representing sales opportunities
 - Links to: Contacts, Companies, Users, Pipelines
 - Contains custom fields and standard fields
 ### 2. Contacts (Контакты)
 - Individual people in the CRM
 - Links to: Companies, Deals, Users
 - Contains communication history
 ### 3. Companies (Компании)
 - Organizations in the CRM
 - Links to: Contacts, Deals, Users
 - Contains company-specific custom fields
 ### 4. Pipelines (Воронки)
 - Sales process definitions
 - Contains stages and automation rules
 - Referenced by Deals
 ### 5. Users (Пользователи)
 - CRM users (employees)
 - Referenced by all other entities as owners/responsible persons
 ### 6. Events (События)
 - Activity logs (filtered types only):
  - `incoming_call`
  - `outgoing_call`
  - `lead_status_changed`
 - Links to related entities
 ## Database Schema
 ### Core Tables
 ```sql
 -- Users table
 CREATE TABLE amo_users (
    id INTEGER PRIMARY KEY,
    name TEXT NOT NULL,
    email TEXT,
    is_active BOOLEAN DEFAULT TRUE,
    created_at INTEGER,
    updated_at INTEGER,
    raw_data JSON -- Original AMO CRM data
 );
 -- Pipelines table
 CREATE TABLE amo_pipelines (
    id INTEGER PRIMARY KEY,
    name TEXT NOT NULL,
    sort INTEGER,
    is_main BOOLEAN DEFAULT FALSE,
    is_unsorted BOOLEAN DEFAULT FALSE,
    is_archive BOOLEAN DEFAULT FALSE,
    account_id INTEGER,
    created_at INTEGER,
    updated_at INTEGER,
    raw_data JSON
 );
 -- Pipeline stages
 CREATE TABLE amo_pipeline_stages (
    id INTEGER PRIMARY KEY,
    pipeline_id INTEGER REFERENCES amo_pipelines(id),
    name TEXT NOT NULL,
    sort INTEGER,
    is_editable BOOLEAN DEFAULT TRUE,
    color TEXT,
    created_at INTEGER,
    updated_at INTEGER,
    raw_data JSON
 );
 -- Companies table
 CREATE TABLE amo_companies (
    id INTEGER PRIMARY KEY,
    name TEXT NOT NULL,
    responsible_user_id INTEGER REFERENCES amo_users(id),
    group_id INTEGER,
    created_by INTEGER REFERENCES amo_users(id),
    updated_by INTEGER REFERENCES amo_users(id),
    created_at INTEGER,
    updated_at INTEGER,
    closest_task_at INTEGER,
    is_deleted BOOLEAN DEFAULT FALSE,
    raw_data JSON
 );
 -- Contacts table
 CREATE TABLE amo_contacts (
    id INTEGER PRIMARY KEY,
    name TEXT NOT NULL,
    first_name TEXT,
    last_name TEXT,
    responsible_user_id INTEGER REFERENCES amo_users(id),
    group_id INTEGER,
    created_by INTEGER REFERENCES amo_users(id),
    updated_by INTEGER REFERENCES amo_users(id),
    created_at INTEGER,
    updated_at INTEGER,
    closest_task_at INTEGER,
    is_deleted BOOLEAN DEFAULT FALSE,
    raw_data JSON
 );
 -- Deals table
 CREATE TABLE amo_deals (
    id INTEGER PRIMARY KEY,
    name TEXT NOT NULL,
    price INTEGER DEFAULT 0,
    responsible_user_id INTEGER REFERENCES amo_users(id),
    group_id INTEGER,
    status_id INTEGER REFERENCES amo_pipeline_stages(id),
    pipeline_id INTEGER REFERENCES amo_pipelines(id),
    loss_reason_id INTEGER,
    created_by INTEGER REFERENCES amo_users(id),
    updated_by INTEGER REFERENCES amo_users(id),
    closed_at INTEGER,
    created_at INTEGER,
    updated_at INTEGER,
    closest_task_at INTEGER,
    is_deleted BOOLEAN DEFAULT FALSE,
    raw_data JSON
 );
 -- Events table
 CREATE TABLE amo_events (
    id INTEGER PRIMARY KEY,
    type TEXT NOT NULL CHECK (type IN ('incoming_call', 'outgoing_call', 'lead_status_changed')),
    entity_id INTEGER,
    entity_type TEXT,
    created_by INTEGER REFERENCES amo_users(id),
    created_at INTEGER,
    value_after JSON,
    value_before JSON,
    account_id INTEGER,
    raw_data JSON
 );
 -- Relations tables
 CREATE TABLE amo_deal_contacts (
    deal_id INTEGER REFERENCES amo_deals(id),
    contact_id INTEGER REFERENCES amo_contacts(id),
    is_main BOOLEAN DEFAULT FALSE,
    PRIMARY KEY (deal_id, contact_id)
 );
 CREATE TABLE amo_deal_companies (
    deal_id INTEGER REFERENCES amo_deals(id),
    company_id INTEGER REFERENCES amo_companies(id),
    is_main BOOLEAN DEFAULT FALSE,
    PRIMARY KEY (deal_id, company_id)
 );
 CREATE TABLE amo_contact_companies (
    contact_id INTEGER REFERENCES amo_contacts(id),
    company_id INTEGER REFERENCES amo_companies(id),
    is_main BOOLEAN DEFAULT FALSE,
    PRIMARY KEY (contact_id, company_id)
 );
 ```
 ### Custom Fields Handling
 ```sql
 -- Universal custom fields table
 CREATE TABLE amo_custom_fields (
    id INTEGER PRIMARY KEY,
    entity_type TEXT NOT NULL, -- 'deals', 'contacts', 'companies'
    entity_id INTEGER NOT NULL,
    field_id INTEGER NOT NULL,
    field_name TEXT NOT NULL,
    field_type TEXT NOT NULL, -- 'text', 'numeric', 'checkbox', 'select', 'multiselect', 'date', 'url', 'textarea'
    field_value TEXT,
    field_value_numeric REAL,
    field_value_date INTEGER,
    is_custom BOOLEAN DEFAULT TRUE,
    created_at INTEGER DEFAULT (strftime('%s', 'now')),
    updated_at INTEGER DEFAULT (strftime('%s', 'now'))
 );
 -- Indexes for performance
 CREATE INDEX idx_custom_fields_entity ON amo_custom_fields(entity_type, entity_id);
 CREATE INDEX idx_custom_fields_field ON amo_custom_fields(field_id);
 CREATE INDEX idx_custom_fields_name ON amo_custom_fields(field_name);
 ```
 ### Google Sheets Export Configuration
 ```sql
 -- Single unified export configuration
 CREATE TABLE export_configuration (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    name TEXT NOT NULL,
    sheet_id TEXT NOT NULL,
    date_range_start INTEGER,
    date_range_end INTEGER,
    is_active BOOLEAN DEFAULT TRUE,
    created_at INTEGER DEFAULT (strftime('%s', 'now')),
    updated_at INTEGER DEFAULT (strftime('%s', 'now'))
 );
 -- Entity-specific sheet mappings within the configuration
 CREATE TABLE export_entity_mappings (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    configuration_id INTEGER REFERENCES export_configuration(id),
    entity_type TEXT NOT NULL, -- 'deals', 'contacts', 'companies', 'pipelines', 'users', 'events'
    sheet_name TEXT NOT NULL, -- Sheet tab name for this entity
    field_mapping JSON NOT NULL, -- [{"field_name": "name", "column": "A", "order": 1}]
    is_enabled BOOLEAN DEFAULT TRUE,
    created_at INTEGER DEFAULT (strftime('%s', 'now')),
    updated_at INTEGER DEFAULT (strftime('%s', 'now'))
 );
 -- Export jobs tracking
 CREATE TABLE export_jobs (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    configuration_id INTEGER REFERENCES export_configuration(id),
    status TEXT NOT NULL DEFAULT 'pending', -- 'pending', 'running', 'completed', 'failed'
    records_processed INTEGER DEFAULT 0,
    total_records INTEGER DEFAULT 0,
    error_message TEXT,
    started_at INTEGER,
    completed_at INTEGER,
    created_at INTEGER DEFAULT (strftime('%s', 'now'))
 );
 ```
 ## API Endpoints
 **Base URL**: `http://localhost:8000/api/v1`
 **Authentication**: None required
 ### 1. Entity Management
 #### GET /entities
 List all entities with statistics
 ```json
 {
  "entities": [
    {
      "type": "deals",
      "count": 1500,
      "last_updated": "2024-01-15T10:30:00Z",
      "date_range": {
        "earliest": "2017-01-01T00:00:00Z",
        "latest": "2024-01-15T10:30:00Z"
      }
    },
    {
      "type": "contacts",
      "count": 3200,
      "last_updated": "2024-01-15T09:45:00Z",
      "date_range": {
        "earliest": "2017-03-15T00:00:00Z",
        "latest": "2024-01-15T09:45:00Z"
      }
    }
  ]
 }
 ```
 #### GET /entities/{entity_type}/fields
 List fields with statistics and examples
 ```json
 {
  "entity_type": "deals",
  "fields": [
    {
      "name": "name",
      "type": "text",
      "is_custom": false,
      "usage_count": 1500,
      "examples": ["Deal #1", "Important client deal", "Q4 opportunity"],
      "null_count": 0
    },
    {
      "name": "custom_priority",
      "type": "select",
      "is_custom": true,
      "field_id": 123456,
      "usage_count": 1200,
      "examples": ["High", "Medium", "Low"],
      "null_count": 300,
      "possible_values": ["High", "Medium", "Low"]
    }
  ]
 }
 ```
 ### 2. AMO CRM Integration
 #### GET /amocrm/info
 Get AMO CRM connection information
 ```json
 {
  "domain": "wecheap.amocrm.ru",
  "has_token": true,
  "base_url": "https://wecheap.amocrm.ru/api/v4",
  "supported_entities": [
    "users", "pipelines", "companies",
    "contacts", "deals", "events"
  ]
 }
 ```
 #### GET /amocrm/fetch/{entity_type}
 Fetch data directly from AMO CRM API
 - **Parameters**: `limit` (default: 10), `page` (default: 1)
 - **Supported entities**: users, pipelines, companies, contacts, deals, events
 - **Returns**: Raw AMO CRM HAL+JSON response
 #### GET /amocrm/fetch/custom_fields/{entity_type}
 Fetch custom fields metadata from AMO CRM
 - **Supported entities**: leads, contacts, companies
 - **Returns**: Custom fields definitions with enums
 #### GET /amocrm/fetch/all
 Fetch all data from AMO CRM (limited amounts for testing)
 - **Returns**: Complete data set from all entities
 #### POST /amocrm/sync/{entity_type}
 Fetch data from AMO CRM and store it in database
 - **Parameters**: `limit` (default: 250), `page` (default: 1)
 - **Returns**: Sync status and fetched data
 ### 3. Google Sheets Export
 #### POST /export/configure
 Create or update unified export configuration for all entities
 ```json
 {
  "name": "Q1 2024 AMO CRM Export",
  "sheet_id": "1BxiMVs0XRA5nFMdKvBdBZjgmUUqptlbs74OgvE2upms",
  "date_range_start": "2024-01-01T00:00:00Z",
  "date_range_end": "2024-03-31T23:59:59Z",
  "entity_mappings": {
    "deals": {
      "sheet_name": "Deals",
      "is_enabled": true,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "price", "column": "B", "order": 2},
        {"field_name": "created_at", "column": "C", "order": 3},
        {"field_name": "custom_priority", "column": "D", "order": 4}
      ]
    },
    "contacts": {
      "sheet_name": "Contacts",
      "is_enabled": true,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "first_name", "column": "B", "order": 2},
        {"field_name": "last_name", "column": "C", "order": 3},
        {"field_name": "created_at", "column": "D", "order": 4}
      ]
    },
    "companies": {
      "sheet_name": "Companies",
      "is_enabled": true,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "created_at", "column": "B", "order": 2}
      ]
    },
    "pipelines": {
      "sheet_name": "Pipelines",
      "is_enabled": false,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "is_main", "column": "B", "order": 2}
      ]
    },
    "users": {
      "sheet_name": "Users",
      "is_enabled": false,
      "field_mapping": [
        {"field_name": "name", "column": "A", "order": 1},
        {"field_name": "email", "column": "B", "order": 2}
      ]
    },
    "events": {
      "sheet_name": "Events",
      "is_enabled": true,
      "field_mapping": [
        {"field_name": "type", "column": "A", "order": 1},
        {"field_name": "entity_type", "column": "B", "order": 2},
        {"field_name": "created_at", "column": "C", "order": 3}
      ]
    }
  }
 }
 ```
 #### POST /export/start
 Start async export job
 ```json
 {
  "configuration_id": 1
 }
 ```
 Response:
 ```json
 {
  "job_id": 42,
  "status": "pending",
  "message": "Export job queued successfully"
 }
 ```
 #### GET /export/status/{job_id}
 Get export job status
 ```json
 {
  "job_id": 42,
  "status": "running",
  "progress": {
    "records_processed": 2100,
    "total_records": 4200,
    "percentage": 50,
    "entities": {
      "deals": {"processed": 1500, "total": 1500, "status": "completed"},
      "contacts": {"processed": 600, "total": 2200, "status": "running"},
      "companies": {"processed": 0, "total": 300, "status": "pending"},
      "events": {"processed": 0, "total": 200, "status": "pending"}
    }
  },
  "started_at": "2024-01-15T10:30:00Z",
  "estimated_completion": "2024-01-15T10:35:00Z"
 }
 ```
 #### GET /export/jobs
 List all export jobs
 ```json
 {
  "jobs": [
    {
      "job_id": 42,
      "configuration_name": "Q1 2024 AMO CRM Export",
      "status": "completed",
      "records_processed": 4200,
      "total_records": 4200,
      "entities_processed": {
        "deals": 1500,
        "contacts": 2200,
        "companies": 300,
        "events": 200
      },
      "started_at": "2024-01-15T10:30:00Z",
      "completed_at": "2024-01-15T10:35:00Z"
    }
  ],
  "total": 1,
  "page": 1,
  "per_page": 20
 }
 ```
 ### 4. Data Ingestion (Worker API)
 #### POST /data/{entity_type}
 Put data into database (used by workers)
 ```json
 {
  "data": [
    {
      "id": 123456,
      "name": "Important Deal",
      "price": 50000,
      "responsible_user_id": 789,
      "pipeline_id": 456,
      "status_id": 142,
      "created_at": 1640995200,
      "updated_at": 1704067200,
      "custom_fields": [
        {
          "field_id": 123456,
          "field_name": "Priority",
          "field_type": "select",
          "values": [{"value": "High", "enum_id": 1}]
        }
      ],
      "_embedded": {
        "contacts": [{"id": 111, "is_main": true}],
        "companies": [{"id": 222, "is_main": true}]
      }
    }
  ],
  "sync_mode": "upsert" // "insert", "upsert", "replace"
 }
 ```
 ## Data Processing Logic
 ### 1. HAL+JSON Parsing
 - Extract main entity data from root level
 - Process `_embedded` section for related entities
 - Handle `_links` section for additional relationships
 - Parse custom fields from `custom_fields_values` array
 ### 2. Date Inference
 - Convert Unix timestamps to readable dates
 - Validate dates are between 2017-2026
 - Handle null/missing dates appropriately
 ### 3. Custom Fields Processing
 ```python
 def process_custom_field(field_data, entity_type, entity_id):
    """
    Process custom field from AMO CRM format
    """
    field_record = {
        'entity_type': entity_type,
        'entity_id': entity_id,
        'field_id': field_data['field_id'],
        'field_name': field_data['field_name'],
        'field_type': field_data['field_type'],
        'is_custom': True
    }
    # Handle different field types
    if field_data['field_type'] in ['text', 'textarea', 'url']:
        field_record['field_value'] = field_data['values'][0]['value']
    elif field_data['field_type'] == 'numeric':
        field_record['field_value_numeric'] = float(field_data['values'][0]['value'])
        field_record['field_value'] = str(field_data['values'][0]['value'])
    elif field_data['field_type'] == 'date':
        field_record['field_value_date'] = int(field_data['values'][0]['value'])
        field_record['field_value'] = str(field_data['values'][0]['value'])
    elif field_data['field_type'] in ['select', 'multiselect']:
        # Handle enum values
        values = [v['value'] for v in field_data['values']]
        field_record['field_value'] = ', '.join(values) if len(values) > 1 else values[0]
    return field_record
 ```
 ### 4. Relationship Mapping
 - Extract relationships from `_embedded` section
 - Create junction table records for many-to-many relationships
 - Handle `is_main` flags for primary relationships
 ## Implementation Status
 ### ✅ Completed Features
 #### Phase 1: Core Infrastructure
 - ✅ FastAPI application structure with uv
 - ✅ SQLite adapter with migrations (Alembic)
 - ✅ Complete entity models with relationships
 - ✅ AMO CRM API client for wecheap.amocrm.ru
 #### Phase 2: Data Ingestion
 - ✅ HAL+JSON parser implementation
 - ✅ Data ingestion endpoints for all entities
 - ✅ Custom fields processing with type detection
 - ✅ Relationship mapping (many-to-many)
 #### Phase 3: API Development
 - ✅ Entity listing endpoints with statistics
 - ✅ Field statistics endpoints with examples
 - ✅ Export configuration management
 - ✅ Direct AMO CRM integration endpoints
 #### Testing & Utilities
 - ✅ Real AMO CRM response examples
 - ✅ Comprehensive test suite
 - ✅ Data fetching script for real responses
 - ✅ API documentation (OpenAPI/Swagger)
 ### 🔄 In Progress / TODO
 #### Phase 4: Google Sheets Integration
 - 🔄 Google Sheets API client implementation
 - 🔄 Actual export processing
 - ✅ Export job management structure
 - ✅ Job status tracking
 #### Phase 5: Workers & Background Processing
 - 🔄 Celery worker implementation
 - 🔄 Data synchronization workers
 - 🔄 Error handling and retry logic
 - 🔄 Monitoring and logging
 ## Technology Stack
 - **Framework**: FastAPI (with uvicorn)
 - **Package Manager**: uv (modern Python package management)
 - **Database**: SQLite with SQLAlchemy 2.0 ORM
 - **Migrations**: Alembic
 - **HTTP Client**: httpx for AMO CRM API
 - **Data Validation**: Pydantic v2
 - **Testing**: pytest with real AMO CRM fixtures
 - **Background Jobs**: Celery with Redis (planned)
 - **Google Sheets**: google-api-python-client (planned)
 - **Development**: Black, isort, mypy, flake8
 ## Configuration
 ### Environment Variables
 ```env
 # Database
 DATABASE_URL=sqlite:///./amo_data.db
 # AMO CRM API (wecheap.amocrm.ru)
 AMO_CRM_DOMAIN=wecheap.amocrm.ru
 AMO_CRM_ACCESS_TOKEN=your-longterm-access-token
 # Google Sheets API (optional)
 GOOGLE_SERVICE_ACCOUNT_FILE=path/to/service-account.json
 GOOGLE_SCOPES=https://www.googleapis.com/auth/spreadsheets
 # Redis (for future Celery integration)
 REDIS_URL=redis://localhost:6379/0
 # API Settings
 API_V1_STR=/api/v1
 LOG_LEVEL=INFO
 ```
 ### Key Features
 - **No Authentication**: Simple public API
 - **Long-term Tokens**: Uses AMO CRM long-term access tokens
 - **Real-time Testing**: Direct AMO CRM data fetching
 - **Comprehensive Fixtures**: Real API response examples
 ## Quick Start
 ### 1. Setup
 ```bash
 # Clone and setup
 git clone <repository>
 cd amo-server
 uv sync
 # Configure environment
 cp env.example .env
 # Edit .env with your AMO CRM token
 ```
 ### 2. Run Service
 ```bash
 # Start development server
 uv run uvicorn app:app --reload
 # Access API documentation
 # http://localhost:8000/docs
 ```
 ### 3. Test AMO CRM Connection
 ```bash
 # Check connection
 curl http://localhost:8000/api/v1/amocrm/info
 # Fetch real data
 curl http://localhost:8000/api/v1/amocrm/fetch/users?limit=5
 # Get all data for testing
 uv run python scripts/fetch_amocrm_data.py
 ```
 ### 4. Example API Usage
 ```bash
 # List entities
 curl http://localhost:8000/api/v1/entities/
 # Get deal fields
 curl http://localhost:8000/api/v1/entities/deals/fields
 # Fetch deals from AMO CRM
 curl http://localhost:8000/api/v1/amocrm/fetch/deals?limit=10
 # Create export configuration
 curl -X POST http://localhost:8000/api/v1/export/configure \
  -H "Content-Type: application/json" \
  -d '{"name": "Test Export", "sheet_id": "123", "entity_mappings": {...}}'
 ```
 ## Security Considerations
 1. **API Keys**: Store AMO CRM tokens securely in environment variables
 2. **No Authentication**: Current API has no authentication (as requested)
 3. **Rate Limiting**: Consider implementing rate limiting for production
 4. **Data Validation**: All incoming data is validated with Pydantic
 5. **CORS**: Currently allows all origins (configure for production)
 ## Performance Considerations
 1. **Database Indexing**: Proper indexes on foreign keys and search fields
 2. **Batch Processing**: Process data in batches to avoid memory issues
 3. **Caching**: Use Redis for frequently accessed data
 4. **Async Processing**: Use async/await for I/O operations
 5. **Pagination**: Implement cursor-based pagination for large datasets
 ## Monitoring & Logging
 1. **Application Metrics**: Track API response times, error rates
 2. **Data Metrics**: Monitor data freshness, sync status
 3. **Export Metrics**: Track export job success rates, processing times
 4. **Error Tracking**: Comprehensive error logging and alerting
 5. **Health Checks**: Implement health check endpoints
 ## Error Handling
 1. **AMO CRM API Errors**: Handle rate limits, timeouts, authentication errors
 2. **Google Sheets API Errors**: Handle quota limits, permission errors (planned)
 3. **Data Validation Errors**: Proper error messages for invalid data
 4. **Database Errors**: Handle connection issues, constraint violations
 5. **Retry Logic**: Exponential backoff for transient failures (planned)
 ## Scripts and Utilities
 ### fetch_amocrm_data.py
 - **Purpose**: Fetch real AMO CRM responses for testing
 - **Usage**: `uv run python scripts/fetch_amocrm_data.py`
 - **Output**:
  - Individual JSON files in `tests/fixtures/real_responses/`
  - Python fixtures in `tests/fixtures/real_amocrm_responses.py`
  - Summary of fetched data
 ### Features
 - Fetches all entity types from wecheap.amocrm.ru
 - Includes custom fields metadata
 - Proper error handling with troubleshooting tips
 - Generates both JSON and Python fixtures
 ## Testing
 ### Test Structure
 - **Unit Tests**: `tests/test_amocrm_examples.py` - Test AMO CRM response structure
 - **API Tests**: `tests/test_api.py` - Test all API endpoints with real data
 - **Fixtures**: Real AMO CRM responses in multiple formats
 ### Running Tests
 ```bash
 # Run all tests
 uv run pytest
 # Run with coverage
 uv run pytest --cov=. --cov-report=html
 # Run specific test file
 uv run pytest tests/test_api.py -v
 ```
 ## Development Workflow
 ### Code Quality
 ```bash
 # Format code
 uv run black .
 uv run isort .
 # Lint code
 uv run flake8
 # Type checking
 uv run mypy .
 ```
 ### Database Migrations
 ```bash
 # Generate migration
 alembic revision --autogenerate -m "Description"
 # Apply migrations
 alembic upgrade head
 ```
--- a/env.docker.example
+++ b/env.docker.example
@ -0,0 +1,10 @@
 # Example environment file for Docker Compose
 # Copy this to .env and fill in your actual values
 # AMO CRM API
 AMO_CRM_DOMAIN=your-domain.amocrm.ru
 AMO_CRM_ACCESS_TOKEN=your-longterm-access-token
 # Google Sheets API
 GOOGLE_SERVICE_ACCOUNT_FILE=/app/credentials/service-account.json
 GOOGLE_SCOPES=https://www.googleapis.com/auth/spreadsheets
--- a/env.example
+++ b/env.example
@ -0,0 +1,19 @@
 # Database
 DATABASE_URL=sqlite:///./amo_data.db
 # AMO CRM API
 AMO_CRM_DOMAIN=wecheap.amocrm.ru
 AMO_CRM_ACCESS_TOKEN=your-longterm-access-token
 # Google Sheets API
 GOOGLE_SERVICE_ACCOUNT_FILE=path/to/service-account.json
 GOOGLE_SCOPES=https://www.googleapis.com/auth/spreadsheets
 # Redis (for Celery)
 REDIS_URL=redis://localhost:6379/0
 # API Settings
 API_V1_STR=/api/v1
 # Logging
 LOG_LEVEL=INFO
--- a/pyproject.toml
+++ b/pyproject.toml
@ -0,0 +1,83 @@
 [project]
 name = "amo-server"
 version = "0.1.0"
 description = "AMO CRM data collection service"
 authors = [
    {name = "Developer", email = "dev@example.com"}
 ]
 dependencies = [
    "fastapi>=0.104.0",
    "uvicorn[standard]>=0.24.0",
    "sqlalchemy>=2.0.0",
    "alembic>=1.13.0",
    "pydantic>=2.5.0",
    "pydantic-settings>=2.1.0",
    "httpx>=0.25.0",
    "python-multipart>=0.0.6",
    "google-api-python-client>=2.100.0",
    "google-auth-httplib2>=0.2.0",
    "google-auth-oauthlib>=1.1.0",
    "celery>=5.3.0",
    "redis>=5.0.0",
    "python-dotenv>=1.0.0",
 ]
 requires-python = ">=3.12"
 readme = "README.md"
 license = {text = "MIT"}
 [project.optional-dependencies]
 dev = [
    "pytest>=7.4.0",
    "pytest-asyncio>=0.21.0",
    "pytest-cov>=4.1.0",
    "black>=23.9.0",
    "isort>=5.12.0",
    "flake8>=6.1.0",
    "mypy>=1.6.0",
    "pre-commit>=3.5.0",
 ]
 [build-system]
 requires = ["hatchling"]
 build-backend = "hatchling.build"
 [tool.hatch.build.targets.wheel]
 packages = ["adapters", "routers", "servers", "utils", "workers"]
 [tool.black]
 line-length = 88
 target-version = ['py312']
 include = '\.pyi?$'
 extend-exclude = '''
 /(
  # directories
  \.eggs
  | \.git
  | \.hg
  | \.mypy_cache
  | \.tox
  | \.venv
  | build
  | dist
 )/
 '''
 [tool.isort]
 profile = "black"
 multi_line_output = 3
 line_length = 88
 known_first_party = ["adapters", "routers", "servers", "utils", "workers"]
 [tool.mypy]
 python_version = "3.12"
 check_untyped_defs = true
 disallow_untyped_defs = true
 disallow_incomplete_defs = true
 warn_redundant_casts = true
 warn_unused_ignores = true
 strict_optional = true
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 asyncio_mode = "auto"
 addopts = "--cov=. --cov-report=term-missing --cov-report=html"
--- a/routers/init.py
+++ b/routers/init.py
@ -0,0 +1 @@
 # Routers package
--- a/routers/amocrm.py
+++ b/routers/amocrm.py
@ -0,0 +1,121 @@
 from fastapi import APIRouter, HTTPException
 from typing import Dict, Any, Optional, List
 import asyncio
 from adapters.amocrm_client import AmoCRMClient
 router = APIRouter()
@router.get("/fetch/{entity_type}")
 async def fetch_amocrm_entity(
    entity_type: str,
    limit: int = 10,
    page: int = 1
 ) -> Dict[str, Any]:
    """Fetch data directly from AMO CRM API"""
    try:
        client = AmoCRMClient()
        if entity_type == "users":
            return await client.get_users(limit=limit)
        elif entity_type == "pipelines":
            return await client.get_pipelines()
        elif entity_type == "companies":
            return await client.get_companies(limit=limit, page=page)
        elif entity_type == "contacts":
            return await client.get_contacts(limit=limit, page=page)
        elif entity_type == "deals":
            return await client.get_deals(limit=limit, page=page)
        elif entity_type == "events":
            return await client.get_events(limit=limit, page=page)
        else:
            raise HTTPException(status_code=404, detail=f"Entity type '{entity_type}' not supported")
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error fetching from AMO CRM: {str(e)}")
@router.get("/fetch/custom_fields/{entity_type}")
 async def fetch_custom_fields(entity_type: str) -> Dict[str, Any]:
    """Fetch custom fields metadata from AMO CRM"""
    valid_types = ["leads", "contacts", "companies"]
    if entity_type not in valid_types:
        raise HTTPException(status_code=400, detail=f"Entity type must be one of: {valid_types}")
    try:
        client = AmoCRMClient()
        return await client.get_custom_fields(entity_type)
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error fetching custom fields: {str(e)}")
@router.get("/fetch/all")
 async def fetch_all_amocrm_data() -> Dict[str, Any]:
    """Fetch all data from AMO CRM (limited amounts for testing)"""
    try:
        client = AmoCRMClient()
        return await client.fetch_all_data()
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error fetching all data: {str(e)}")
@router.post("/sync/{entity_type}")
 async def sync_entity_from_amocrm(
    entity_type: str,
    limit: int = 250,
    page: int = 1
 ) -> Dict[str, Any]:
    """Fetch data from AMO CRM and store it in database"""
    # This would integrate with the data ingestion endpoints
    # For now, just fetch the data
    try:
        client = AmoCRMClient()
        if entity_type == "users":
            data = await client.get_users(limit=limit)
        elif entity_type == "pipelines":
            data = await client.get_pipelines()
        elif entity_type == "companies":
            data = await client.get_companies(limit=limit, page=page)
        elif entity_type == "contacts":
            data = await client.get_contacts(limit=limit, page=page)
        elif entity_type == "deals":
            data = await client.get_deals(limit=limit, page=page)
        elif entity_type == "events":
            data = await client.get_events(limit=limit, page=page)
        else:
            raise HTTPException(status_code=404, detail=f"Entity type '{entity_type}' not supported")
        # TODO: Integrate with data ingestion endpoints
        # For now, return the fetched data
        return {
            "message": f"Fetched {entity_type} from AMO CRM",
            "data": data,
            "sync_status": "fetched_only"  # Would be "synced" when integrated
        }
    except Exception as e:
        raise HTTPException(status_code=500, detail=f"Error syncing from AMO CRM: {str(e)}")
@router.get("/info")
 async def amocrm_info() -> Dict[str, Any]:
    """Get AMO CRM connection info"""
    from utils.config import settings
    return {
        "domain": settings.AMO_CRM_DOMAIN,
        "has_token": bool(settings.AMO_CRM_ACCESS_TOKEN),
        "base_url": f"https://{settings.AMO_CRM_DOMAIN}/api/v4",
        "supported_entities": [
            "users", "pipelines", "companies",
            "contacts", "deals", "events"
        ]
    }
--- a/routers/data.py
+++ b/routers/data.py
@ -0,0 +1,528 @@
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.orm import Session
 from typing import Dict, Any, List, Optional, Union
 from pydantic import BaseModel
 from datetime import datetime
 import time
 import json
 from adapters.sqlite.database import get_db
 from adapters.sqlite.models import (
    Deal, Contact, Company, Pipeline, PipelineStage, User, Event, CustomField,
    deal_contacts, deal_companies, contact_companies
 )
 router = APIRouter()
 class CustomFieldValue(BaseModel):
    field_id: int
    field_name: str
    field_type: str
    values: List[Dict[str, Any]]
 class EmbeddedRelation(BaseModel):
    id: int
    is_main: Optional[bool] = None
 class DataItem(BaseModel):
    id: int
    name: Optional[str] = None
    # Add other common fields as needed
    created_at: Optional[int] = None
    updated_at: Optional[int] = None
    custom_fields_values: Optional[List[CustomFieldValue]] = None
    _embedded: Optional[Dict[str, List[EmbeddedRelation]]] = None
    # Store raw data for complete preservation
    raw_data: Optional[Dict[str, Any]] = None
 class DataIngestionRequest(BaseModel):
    data: List[Dict[str, Any]]
    sync_mode: str = "upsert"  # "insert", "upsert", "replace"
 def process_custom_fields(
    custom_fields: List[Dict[str, Any]],
    entity_type: str,
    entity_id: int,
    db: Session
 ):
    """Process and store custom fields"""
    # Clear existing custom fields for this entity if replacing
    db.query(CustomField).filter(
        CustomField.entity_type == entity_type,
        CustomField.entity_id == entity_id
    ).delete()
    for field_data in custom_fields:
        field_id = field_data.get('field_id')
        field_name = field_data.get('field_name', '')
        field_type = field_data.get('field_type', 'text')
        values = field_data.get('values', [])
        for value_data in values:
            value = value_data.get('value', '')
            custom_field = CustomField(
                entity_type=entity_type,
                entity_id=entity_id,
                field_id=field_id,
                field_name=field_name,
                field_type=field_type,
                field_value=str(value) if value else None,
                is_custom=True,
                created_at=int(time.time()),
                updated_at=int(time.time())
            )
            # Handle different field types
            if field_type == 'numeric' and value:
                try:
                    custom_field.field_value_numeric = float(value)
                except (ValueError, TypeError):
                    pass
            elif field_type == 'date' and value:
                try:
                    # AMO CRM returns dates as unix timestamps
                    custom_field.field_value_date = int(value)
                except (ValueError, TypeError):
                    pass
            elif field_type in ['select', 'multiselect']:
                # For select fields, store the display value
                custom_field.field_value = str(value)
            db.add(custom_field)
 def process_relationships(
    embedded_data: Dict[str, Any],
    entity_id: int,
    entity_type: str,
    db: Session
 ):
    """Process embedded relationships"""
    if entity_type == "deals":
        # Process deal-contact relationships
        if 'contacts' in embedded_data:
            # Clear existing relationships
            db.execute(
                deal_contacts.delete().where(deal_contacts.c.deal_id == entity_id)
            )
            for contact_data in embedded_data['contacts']:
                contact_id = contact_data['id']
                is_main = contact_data.get('is_main', False)
                # Insert relationship
                db.execute(
                    deal_contacts.insert().values(
                        deal_id=entity_id,
                        contact_id=contact_id,
                        is_main=is_main
                    )
                )
        # Process deal-company relationships
        if 'companies' in embedded_data:
            # Clear existing relationships
            db.execute(
                deal_companies.delete().where(deal_companies.c.deal_id == entity_id)
            )
            for company_data in embedded_data['companies']:
                company_id = company_data['id']
                is_main = company_data.get('is_main', False)
                # Insert relationship
                db.execute(
                    deal_companies.insert().values(
                        deal_id=entity_id,
                        company_id=company_id,
                        is_main=is_main
                    )
                )
    elif entity_type == "contacts":
        # Process contact-company relationships
        if 'companies' in embedded_data:
            # Clear existing relationships
            db.execute(
                contact_companies.delete().where(contact_companies.c.contact_id == entity_id)
            )
            for company_data in embedded_data['companies']:
                company_id = company_data['id']
                is_main = company_data.get('is_main', False)
                # Insert relationship
                db.execute(
                    contact_companies.insert().values(
                        contact_id=entity_id,
                        company_id=company_id,
                        is_main=is_main
                    )
                )
@router.post("/users")
 async def put_users_data(
    request: DataIngestionRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Put users data into database"""
    processed_count = 0
    for user_data in request.data:
        user_id = user_data['id']
        # Check if user exists
        existing_user = db.query(User).filter(User.id == user_id).first()
        if request.sync_mode == "insert" and existing_user:
            continue  # Skip existing records in insert mode
        user_values = {
            'id': user_id,
            'name': user_data.get('name'),
            'email': user_data.get('email'),
            'is_active': user_data.get('is_active', True),
            'created_at': user_data.get('created_at'),
            'updated_at': user_data.get('updated_at', int(time.time())),
            'raw_data': user_data
        }
        if existing_user:
            # Update existing user
            for key, value in user_values.items():
                if key != 'id':  # Don't update ID
                    setattr(existing_user, key, value)
        else:
            # Create new user
            new_user = User(**user_values)
            db.add(new_user)
        processed_count += 1
    db.commit()
    return {
        "message": f"Processed {processed_count} users",
        "processed_count": processed_count
    }
@router.post("/pipelines")
 async def put_pipelines_data(
    request: DataIngestionRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Put pipelines data into database"""
    processed_count = 0
    for pipeline_data in request.data:
        pipeline_id = pipeline_data['id']
        # Check if pipeline exists
        existing_pipeline = db.query(Pipeline).filter(Pipeline.id == pipeline_id).first()
        if request.sync_mode == "insert" and existing_pipeline:
            continue
        pipeline_values = {
            'id': pipeline_id,
            'name': pipeline_data.get('name'),
            'sort': pipeline_data.get('sort'),
            'is_main': pipeline_data.get('is_main', False),
            'is_unsorted': pipeline_data.get('is_unsorted', False),
            'is_archive': pipeline_data.get('is_archive', False),
            'account_id': pipeline_data.get('account_id'),
            'created_at': pipeline_data.get('created_at'),
            'updated_at': pipeline_data.get('updated_at', int(time.time())),
            'raw_data': pipeline_data
        }
        if existing_pipeline:
            for key, value in pipeline_values.items():
                if key != 'id':
                    setattr(existing_pipeline, key, value)
        else:
            new_pipeline = Pipeline(**pipeline_values)
            db.add(new_pipeline)
        # Process pipeline stages
        embedded_data = pipeline_data.get('_embedded', {})
        if 'statuses' in embedded_data:
            for status_data in embedded_data['statuses']:
                status_id = status_data['id']
                existing_stage = db.query(PipelineStage).filter(PipelineStage.id == status_id).first()
                stage_values = {
                    'id': status_id,
                    'pipeline_id': pipeline_id,
                    'name': status_data.get('name'),
                    'sort': status_data.get('sort'),
                    'is_editable': status_data.get('is_editable', True),
                    'color': status_data.get('color'),
                    'created_at': status_data.get('created_at'),
                    'updated_at': status_data.get('updated_at', int(time.time())),
                    'raw_data': status_data
                }
                if existing_stage:
                    for key, value in stage_values.items():
                        if key != 'id':
                            setattr(existing_stage, key, value)
                else:
                    new_stage = PipelineStage(**stage_values)
                    db.add(new_stage)
        processed_count += 1
    db.commit()
    return {
        "message": f"Processed {processed_count} pipelines",
        "processed_count": processed_count
    }
@router.post("/companies")
 async def put_companies_data(
    request: DataIngestionRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Put companies data into database"""
    processed_count = 0
    for company_data in request.data:
        company_id = company_data['id']
        existing_company = db.query(Company).filter(Company.id == company_id).first()
        if request.sync_mode == "insert" and existing_company:
            continue
        company_values = {
            'id': company_id,
            'name': company_data.get('name'),
            'responsible_user_id': company_data.get('responsible_user_id'),
            'group_id': company_data.get('group_id'),
            'created_by': company_data.get('created_by'),
            'updated_by': company_data.get('updated_by'),
            'created_at': company_data.get('created_at'),
            'updated_at': company_data.get('updated_at', int(time.time())),
            'closest_task_at': company_data.get('closest_task_at'),
            'is_deleted': company_data.get('is_deleted', False),
            'raw_data': company_data
        }
        if existing_company:
            for key, value in company_values.items():
                if key != 'id':
                    setattr(existing_company, key, value)
        else:
            new_company = Company(**company_values)
            db.add(new_company)
        # Process custom fields
        custom_fields = company_data.get('custom_fields_values', [])
        if custom_fields:
            process_custom_fields(custom_fields, "companies", company_id, db)
        processed_count += 1
    db.commit()
    return {
        "message": f"Processed {processed_count} companies",
        "processed_count": processed_count
    }
@router.post("/contacts")
 async def put_contacts_data(
    request: DataIngestionRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Put contacts data into database"""
    processed_count = 0
    for contact_data in request.data:
        contact_id = contact_data['id']
        existing_contact = db.query(Contact).filter(Contact.id == contact_id).first()
        if request.sync_mode == "insert" and existing_contact:
            continue
        contact_values = {
            'id': contact_id,
            'name': contact_data.get('name'),
            'first_name': contact_data.get('first_name'),
            'last_name': contact_data.get('last_name'),
            'responsible_user_id': contact_data.get('responsible_user_id'),
            'group_id': contact_data.get('group_id'),
            'created_by': contact_data.get('created_by'),
            'updated_by': contact_data.get('updated_by'),
            'created_at': contact_data.get('created_at'),
            'updated_at': contact_data.get('updated_at', int(time.time())),
            'closest_task_at': contact_data.get('closest_task_at'),
            'is_deleted': contact_data.get('is_deleted', False),
            'raw_data': contact_data
        }
        if existing_contact:
            for key, value in contact_values.items():
                if key != 'id':
                    setattr(existing_contact, key, value)
        else:
            new_contact = Contact(**contact_values)
            db.add(new_contact)
        # Process custom fields
        custom_fields = contact_data.get('custom_fields_values', [])
        if custom_fields:
            process_custom_fields(custom_fields, "contacts", contact_id, db)
        # Process relationships
        embedded_data = contact_data.get('_embedded', {})
        if embedded_data:
            process_relationships(embedded_data, contact_id, "contacts", db)
        processed_count += 1
    db.commit()
    return {
        "message": f"Processed {processed_count} contacts",
        "processed_count": processed_count
    }
@router.post("/deals")
 async def put_deals_data(
    request: DataIngestionRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Put deals (leads) data into database"""
    processed_count = 0
    for deal_data in request.data:
        deal_id = deal_data['id']
        existing_deal = db.query(Deal).filter(Deal.id == deal_id).first()
        if request.sync_mode == "insert" and existing_deal:
            continue
        deal_values = {
            'id': deal_id,
            'name': deal_data.get('name'),
            'price': deal_data.get('price', 0),
            'responsible_user_id': deal_data.get('responsible_user_id'),
            'group_id': deal_data.get('group_id'),
            'status_id': deal_data.get('status_id'),
            'pipeline_id': deal_data.get('pipeline_id'),
            'loss_reason_id': deal_data.get('loss_reason_id'),
            'created_by': deal_data.get('created_by'),
            'updated_by': deal_data.get('updated_by'),
            'closed_at': deal_data.get('closed_at'),
            'created_at': deal_data.get('created_at'),
            'updated_at': deal_data.get('updated_at', int(time.time())),
            'closest_task_at': deal_data.get('closest_task_at'),
            'is_deleted': deal_data.get('is_deleted', False),
            'raw_data': deal_data
        }
        if existing_deal:
            for key, value in deal_values.items():
                if key != 'id':
                    setattr(existing_deal, key, value)
        else:
            new_deal = Deal(**deal_values)
            db.add(new_deal)
        # Process custom fields
        custom_fields = deal_data.get('custom_fields_values', [])
        if custom_fields:
            process_custom_fields(custom_fields, "deals", deal_id, db)
        # Process relationships
        embedded_data = deal_data.get('_embedded', {})
        if embedded_data:
            process_relationships(embedded_data, deal_id, "deals", db)
        processed_count += 1
    db.commit()
    return {
        "message": f"Processed {processed_count} deals",
        "processed_count": processed_count
    }
@router.post("/events")
 async def put_events_data(
    request: DataIngestionRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Put events data into database"""
    processed_count = 0
    for event_data in request.data:
        event_id = event_data['id']
        event_type = event_data.get('type')
        # Filter only supported event types
        if event_type not in ['incoming_call', 'outgoing_call', 'lead_status_changed']:
            continue
        existing_event = db.query(Event).filter(Event.id == event_id).first()
        if request.sync_mode == "insert" and existing_event:
            continue
        event_values = {
            'id': event_id,
            'type': event_type,
            'entity_id': event_data.get('entity_id'),
            'entity_type': event_data.get('entity_type'),
            'created_by': event_data.get('created_by'),
            'created_at': event_data.get('created_at'),
            'value_after': event_data.get('value_after'),
            'value_before': event_data.get('value_before'),
            'account_id': event_data.get('account_id'),
            'raw_data': event_data
        }
        if existing_event:
            for key, value in event_values.items():
                if key != 'id':
                    setattr(existing_event, key, value)
        else:
            new_event = Event(**event_values)
            db.add(new_event)
        processed_count += 1
    db.commit()
    return {
        "message": f"Processed {processed_count} events",
        "processed_count": processed_count
    }
--- a/routers/entities.py
+++ b/routers/entities.py
@ -0,0 +1,324 @@
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.orm import Session
 from sqlalchemy import func, distinct
 from typing import List, Dict, Any, Optional
 from datetime import datetime
 from adapters.sqlite.database import get_db
 from adapters.sqlite.models import (
    Deal, Contact, Company, Pipeline, User, Event, CustomField
 )
 router = APIRouter()
@router.get("/")
 async def list_entities(db: Session = Depends(get_db)) -> Dict[str, Any]:
    """List all entities with statistics"""
    entities = {}
    # Deals statistics
    deals_count = db.query(Deal).count()
    deals_date_range = db.query(
        func.min(Deal.created_at).label('earliest'),
        func.max(Deal.updated_at).label('latest')
    ).first()
    entities["deals"] = {
        "count": deals_count,
        "last_updated": datetime.fromtimestamp(deals_date_range.latest).isoformat() + "Z" if deals_date_range.latest else None,
        "date_range": {
            "earliest": datetime.fromtimestamp(deals_date_range.earliest).isoformat() + "Z" if deals_date_range.earliest else None,
            "latest": datetime.fromtimestamp(deals_date_range.latest).isoformat() + "Z" if deals_date_range.latest else None
        }
    }
    # Contacts statistics
    contacts_count = db.query(Contact).count()
    contacts_date_range = db.query(
        func.min(Contact.created_at).label('earliest'),
        func.max(Contact.updated_at).label('latest')
    ).first()
    entities["contacts"] = {
        "count": contacts_count,
        "last_updated": datetime.fromtimestamp(contacts_date_range.latest).isoformat() + "Z" if contacts_date_range.latest else None,
        "date_range": {
            "earliest": datetime.fromtimestamp(contacts_date_range.earliest).isoformat() + "Z" if contacts_date_range.earliest else None,
            "latest": datetime.fromtimestamp(contacts_date_range.latest).isoformat() + "Z" if contacts_date_range.latest else None
        }
    }
    # Companies statistics
    companies_count = db.query(Company).count()
    companies_date_range = db.query(
        func.min(Company.created_at).label('earliest'),
        func.max(Company.updated_at).label('latest')
    ).first()
    entities["companies"] = {
        "count": companies_count,
        "last_updated": datetime.fromtimestamp(companies_date_range.latest).isoformat() + "Z" if companies_date_range.latest else None,
        "date_range": {
            "earliest": datetime.fromtimestamp(companies_date_range.earliest).isoformat() + "Z" if companies_date_range.earliest else None,
            "latest": datetime.fromtimestamp(companies_date_range.latest).isoformat() + "Z" if companies_date_range.latest else None
        }
    }
    # Pipelines statistics
    pipelines_count = db.query(Pipeline).count()
    pipelines_date_range = db.query(
        func.min(Pipeline.created_at).label('earliest'),
        func.max(Pipeline.updated_at).label('latest')
    ).first()
    entities["pipelines"] = {
        "count": pipelines_count,
        "last_updated": datetime.fromtimestamp(pipelines_date_range.latest).isoformat() + "Z" if pipelines_date_range.latest else None,
        "date_range": {
            "earliest": datetime.fromtimestamp(pipelines_date_range.earliest).isoformat() + "Z" if pipelines_date_range.earliest else None,
            "latest": datetime.fromtimestamp(pipelines_date_range.latest).isoformat() + "Z" if pipelines_date_range.latest else None
        }
    }
    # Users statistics (no date range as they don't change often)
    users_count = db.query(User).count()
    entities["users"] = {
        "count": users_count,
        "last_updated": None,
        "date_range": {
            "earliest": None,
            "latest": None
        }
    }
    # Events statistics
    events_count = db.query(Event).count()
    events_date_range = db.query(
        func.min(Event.created_at).label('earliest'),
        func.max(Event.created_at).label('latest')
    ).first()
    entities["events"] = {
        "count": events_count,
        "last_updated": datetime.fromtimestamp(events_date_range.latest).isoformat() + "Z" if events_date_range.latest else None,
        "date_range": {
            "earliest": datetime.fromtimestamp(events_date_range.earliest).isoformat() + "Z" if events_date_range.earliest else None,
            "latest": datetime.fromtimestamp(events_date_range.latest).isoformat() + "Z" if events_date_range.latest else None
        }
    }
    return {"entities": entities}
@router.get("/{entity_type}/fields")
 async def list_entity_fields(entity_type: str, db: Session = Depends(get_db)) -> Dict[str, Any]:
    """List fields with statistics and examples for specified entity"""
    if entity_type not in ["deals", "contacts", "companies", "pipelines", "users", "events"]:
        raise HTTPException(status_code=404, detail="Entity type not found")
    # Get standard fields based on entity type
    standard_fields = []
    if entity_type == "deals":
        # Sample some deals to get examples
        sample_deals = db.query(Deal).limit(3).all()
        standard_fields = [
            {
                "name": "name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Deal).filter(Deal.name.isnot(None)).count(),
                "examples": [deal.name for deal in sample_deals if deal.name][:3],
                "null_count": db.query(Deal).filter(Deal.name.is_(None)).count()
            },
            {
                "name": "price",
                "type": "numeric",
                "is_custom": False,
                "usage_count": db.query(Deal).filter(Deal.price.isnot(None)).count(),
                "examples": [deal.price for deal in sample_deals if deal.price is not None][:3],
                "null_count": db.query(Deal).filter(Deal.price.is_(None)).count()
            },
            {
                "name": "created_at",
                "type": "date",
                "is_custom": False,
                "usage_count": db.query(Deal).filter(Deal.created_at.isnot(None)).count(),
                "examples": [datetime.fromtimestamp(deal.created_at).isoformat() + "Z" for deal in sample_deals if deal.created_at][:3],
                "null_count": db.query(Deal).filter(Deal.created_at.is_(None)).count()
            },
            {
                "name": "updated_at",
                "type": "date",
                "is_custom": False,
                "usage_count": db.query(Deal).filter(Deal.updated_at.isnot(None)).count(),
                "examples": [datetime.fromtimestamp(deal.updated_at).isoformat() + "Z" for deal in sample_deals if deal.updated_at][:3],
                "null_count": db.query(Deal).filter(Deal.updated_at.is_(None)).count()
            }
        ]
    elif entity_type == "contacts":
        sample_contacts = db.query(Contact).limit(3).all()
        standard_fields = [
            {
                "name": "name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Contact).filter(Contact.name.isnot(None)).count(),
                "examples": [contact.name for contact in sample_contacts if contact.name][:3],
                "null_count": db.query(Contact).filter(Contact.name.is_(None)).count()
            },
            {
                "name": "first_name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Contact).filter(Contact.first_name.isnot(None)).count(),
                "examples": [contact.first_name for contact in sample_contacts if contact.first_name][:3],
                "null_count": db.query(Contact).filter(Contact.first_name.is_(None)).count()
            },
            {
                "name": "last_name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Contact).filter(Contact.last_name.isnot(None)).count(),
                "examples": [contact.last_name for contact in sample_contacts if contact.last_name][:3],
                "null_count": db.query(Contact).filter(Contact.last_name.is_(None)).count()
            }
        ]
    elif entity_type == "companies":
        sample_companies = db.query(Company).limit(3).all()
        standard_fields = [
            {
                "name": "name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Company).filter(Company.name.isnot(None)).count(),
                "examples": [company.name for company in sample_companies if company.name][:3],
                "null_count": db.query(Company).filter(Company.name.is_(None)).count()
            }
        ]
    elif entity_type == "pipelines":
        sample_pipelines = db.query(Pipeline).limit(3).all()
        standard_fields = [
            {
                "name": "name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Pipeline).filter(Pipeline.name.isnot(None)).count(),
                "examples": [pipeline.name for pipeline in sample_pipelines if pipeline.name][:3],
                "null_count": db.query(Pipeline).filter(Pipeline.name.is_(None)).count()
            },
            {
                "name": "is_main",
                "type": "checkbox",
                "is_custom": False,
                "usage_count": db.query(Pipeline).filter(Pipeline.is_main.isnot(None)).count(),
                "examples": [pipeline.is_main for pipeline in sample_pipelines if pipeline.is_main is not None][:3],
                "null_count": db.query(Pipeline).filter(Pipeline.is_main.is_(None)).count()
            }
        ]
    elif entity_type == "users":
        sample_users = db.query(User).limit(3).all()
        standard_fields = [
            {
                "name": "name",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(User).filter(User.name.isnot(None)).count(),
                "examples": [user.name for user in sample_users if user.name][:3],
                "null_count": db.query(User).filter(User.name.is_(None)).count()
            },
            {
                "name": "email",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(User).filter(User.email.isnot(None)).count(),
                "examples": [user.email for user in sample_users if user.email][:3],
                "null_count": db.query(User).filter(User.email.is_(None)).count()
            }
        ]
    elif entity_type == "events":
        sample_events = db.query(Event).limit(3).all()
        standard_fields = [
            {
                "name": "type",
                "type": "select",
                "is_custom": False,
                "usage_count": db.query(Event).filter(Event.type.isnot(None)).count(),
                "examples": [event.type for event in sample_events if event.type][:3],
                "null_count": db.query(Event).filter(Event.type.is_(None)).count(),
                "possible_values": ["incoming_call", "outgoing_call", "lead_status_changed"]
            },
            {
                "name": "entity_type",
                "type": "text",
                "is_custom": False,
                "usage_count": db.query(Event).filter(Event.entity_type.isnot(None)).count(),
                "examples": [event.entity_type for event in sample_events if event.entity_type][:3],
                "null_count": db.query(Event).filter(Event.entity_type.is_(None)).count()
            }
        ]
    # Get custom fields for this entity type
    custom_fields_query = db.query(CustomField).filter(
        CustomField.entity_type == entity_type
    ).group_by(CustomField.field_name, CustomField.field_type).all()
    custom_fields = []
    for field_group in custom_fields_query:
        # Get examples for this custom field
        examples_query = db.query(CustomField.field_value).filter(
            CustomField.entity_type == entity_type,
            CustomField.field_name == field_group.field_name,
            CustomField.field_value.isnot(None)
        ).limit(3).all()
        examples = [ex[0] for ex in examples_query if ex[0]]
        # Count usage
        usage_count = db.query(CustomField).filter(
            CustomField.entity_type == entity_type,
            CustomField.field_name == field_group.field_name,
            CustomField.field_value.isnot(None)
        ).count()
        null_count = db.query(CustomField).filter(
            CustomField.entity_type == entity_type,
            CustomField.field_name == field_group.field_name,
            CustomField.field_value.is_(None)
        ).count()
        custom_field = {
            "name": field_group.field_name,
            "type": field_group.field_type,
            "is_custom": True,
            "field_id": field_group.field_id,
            "usage_count": usage_count,
            "examples": examples,
            "null_count": null_count
        }
        # For select fields, get possible values
        if field_group.field_type in ["select", "multiselect"]:
            possible_values = db.query(distinct(CustomField.field_value)).filter(
                CustomField.entity_type == entity_type,
                CustomField.field_name == field_group.field_name,
                CustomField.field_value.isnot(None)
            ).all()
            custom_field["possible_values"] = [pv[0] for pv in possible_values if pv[0]]
        custom_fields.append(custom_field)
    all_fields = standard_fields + custom_fields
    return {
        "entity_type": entity_type,
        "fields": all_fields
    }
--- a/routers/export.py
+++ b/routers/export.py
@ -0,0 +1,268 @@
 from fastapi import APIRouter, Depends, HTTPException
 from sqlalchemy.orm import Session
 from typing import Dict, Any, List, Optional
 from pydantic import BaseModel
 from datetime import datetime
 import time
 from adapters.sqlite.database import get_db
 from adapters.sqlite.models import ExportConfiguration, ExportEntityMapping, ExportJob
 router = APIRouter()
 class FieldMapping(BaseModel):
    field_name: str
    column: str
    order: int
 class EntityMappingConfig(BaseModel):
    sheet_name: str
    is_enabled: bool = True
    field_mapping: List[FieldMapping]
 class ExportConfigurationRequest(BaseModel):
    name: str
    sheet_id: str
    date_range_start: Optional[str] = None
    date_range_end: Optional[str] = None
    entity_mappings: Dict[str, EntityMappingConfig]
 class ExportJobStart(BaseModel):
    configuration_id: int
@router.post("/configure")
 async def create_export_configuration(
    config: ExportConfigurationRequest,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Create or update unified export configuration for all entities"""
    # Convert date strings to timestamps
    date_start = None
    date_end = None
    if config.date_range_start:
        try:
            date_start = int(datetime.fromisoformat(config.date_range_start.replace('Z', '+00:00')).timestamp())
        except ValueError:
            raise HTTPException(status_code=400, detail="Invalid date_range_start format")
    if config.date_range_end:
        try:
            date_end = int(datetime.fromisoformat(config.date_range_end.replace('Z', '+00:00')).timestamp())
        except ValueError:
            raise HTTPException(status_code=400, detail="Invalid date_range_end format")
    # Create configuration
    db_config = ExportConfiguration(
        name=config.name,
        sheet_id=config.sheet_id,
        date_range_start=date_start,
        date_range_end=date_end,
        is_active=True,
        created_at=int(time.time()),
        updated_at=int(time.time())
    )
    db.add(db_config)
    db.flush()  # Get the ID
    # Create entity mappings
    valid_entities = ["deals", "contacts", "companies", "pipelines", "users", "events"]
    for entity_type, mapping_config in config.entity_mappings.items():
        if entity_type not in valid_entities:
            raise HTTPException(status_code=400, detail=f"Invalid entity type: {entity_type}")
        # Convert field mapping to JSON
        field_mapping_json = [
            {
                "field_name": fm.field_name,
                "column": fm.column,
                "order": fm.order
            }
            for fm in mapping_config.field_mapping
        ]
        db_mapping = ExportEntityMapping(
            configuration_id=db_config.id,
            entity_type=entity_type,
            sheet_name=mapping_config.sheet_name,
            field_mapping=field_mapping_json,
            is_enabled=mapping_config.is_enabled,
            created_at=int(time.time()),
            updated_at=int(time.time())
        )
        db.add(db_mapping)
    db.commit()
    return {
        "configuration_id": db_config.id,
        "message": "Export configuration created successfully"
    }
@router.get("/configurations")
 async def list_export_configurations(db: Session = Depends(get_db)) -> Dict[str, Any]:
    """List all export configurations"""
    configs = db.query(ExportConfiguration).filter(
        ExportConfiguration.is_active == True
    ).all()
    result = []
    for config in configs:
        entity_mappings = {}
        for mapping in config.entity_mappings:
            entity_mappings[mapping.entity_type] = {
                "sheet_name": mapping.sheet_name,
                "is_enabled": mapping.is_enabled,
                "field_mapping": mapping.field_mapping
            }
        result.append({
            "id": config.id,
            "name": config.name,
            "sheet_id": config.sheet_id,
            "date_range_start": datetime.fromtimestamp(config.date_range_start).isoformat() + "Z" if config.date_range_start else None,
            "date_range_end": datetime.fromtimestamp(config.date_range_end).isoformat() + "Z" if config.date_range_end else None,
            "entity_mappings": entity_mappings,
            "created_at": datetime.fromtimestamp(config.created_at).isoformat() + "Z" if config.created_at else None
        })
    return {"configurations": result}
@router.post("/start")
 async def start_export_job(
    job_request: ExportJobStart,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """Start async export job"""
    # Check if configuration exists
    config = db.query(ExportConfiguration).filter(
        ExportConfiguration.id == job_request.configuration_id,
        ExportConfiguration.is_active == True
    ).first()
    if not config:
        raise HTTPException(status_code=404, detail="Export configuration not found")
    # Create export job
    job = ExportJob(
        configuration_id=config.id,
        status="pending",
        records_processed=0,
        total_records=0,
        created_at=int(time.time())
    )
    db.add(job)
    db.commit()
    # TODO: Queue job for background processing with Celery
    # celery_app.send_task("export_to_sheets", args=[job.id])
    return {
        "job_id": job.id,
        "status": "pending",
        "message": "Export job queued successfully"
    }
@router.get("/status/{job_id}")
 async def get_export_status(job_id: int, db: Session = Depends(get_db)) -> Dict[str, Any]:
    """Get export job status"""
    job = db.query(ExportJob).filter(ExportJob.id == job_id).first()
    if not job:
        raise HTTPException(status_code=404, detail="Export job not found")
    result = {
        "job_id": job.id,
        "status": job.status,
        "progress": {
            "records_processed": job.records_processed,
            "total_records": job.total_records,
            "percentage": round((job.records_processed / job.total_records * 100), 2) if job.total_records > 0 else 0
        }
    }
    if job.started_at:
        result["started_at"] = datetime.fromtimestamp(job.started_at).isoformat() + "Z"
    if job.completed_at:
        result["completed_at"] = datetime.fromtimestamp(job.completed_at).isoformat() + "Z"
    if job.error_message:
        result["error_message"] = job.error_message
    # TODO: Add detailed entity progress when implementing worker
    if job.status == "running":
        result["progress"]["entities"] = {
            "deals": {"processed": 1500, "total": 1500, "status": "completed"},
            "contacts": {"processed": 600, "total": 2200, "status": "running"},
            "companies": {"processed": 0, "total": 300, "status": "pending"},
            "events": {"processed": 0, "total": 200, "status": "pending"}
        }
        result["estimated_completion"] = "2024-01-15T10:35:00Z"  # TODO: Calculate based on progress
    return result
@router.get("/jobs")
 async def list_export_jobs(
    page: int = 1,
    per_page: int = 20,
    db: Session = Depends(get_db)
 ) -> Dict[str, Any]:
    """List all export jobs"""
    offset = (page - 1) * per_page
    jobs_query = db.query(ExportJob).order_by(ExportJob.created_at.desc())
    total = jobs_query.count()
    jobs = jobs_query.offset(offset).limit(per_page).all()
    result_jobs = []
    for job in jobs:
        job_data = {
            "job_id": job.id,
            "configuration_name": job.configuration.name,
            "status": job.status,
            "records_processed": job.records_processed,
            "total_records": job.total_records
        }
        if job.started_at:
            job_data["started_at"] = datetime.fromtimestamp(job.started_at).isoformat() + "Z"
        if job.completed_at:
            job_data["completed_at"] = datetime.fromtimestamp(job.completed_at).isoformat() + "Z"
        # TODO: Add entities_processed breakdown when implementing worker
        if job.status == "completed":
            job_data["entities_processed"] = {
                "deals": 1500,
                "contacts": 2200,
                "companies": 300,
                "events": 200
            }
        result_jobs.append(job_data)
    return {
        "jobs": result_jobs,
        "total": total,
        "page": page,
        "per_page": per_page
    }
--- a/scripts/fetch_amocrm_data.py
+++ b/scripts/fetch_amocrm_data.py
@ -0,0 +1,131 @@
 #!/usr/bin/env python3
 """
 Script to fetch real AMO CRM responses for testing
 Usage: python scripts/fetch_amocrm_data.py
 """
 import asyncio
 import json
 import os
 import sys
 from datetime import datetime
 from pathlib import Path
 # Add project root to Python path
 project_root = Path(__file__).parent.parent
 sys.path.insert(0, str(project_root))
 from adapters.amocrm_client import AmoCRMClient
 from utils.config import settings
 def save_response_to_file(data: dict, filename: str, output_dir: Path = None):
    """Save response data to a JSON file"""
    if output_dir is None:
        output_dir = project_root / "tests" / "fixtures" / "real_responses"
    output_dir.mkdir(parents=True, exist_ok=True)
    filepath = output_dir / f"{filename}.json"
    with open(filepath, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=2)
    print(f"💾 Saved {filename} to {filepath}")
 def generate_python_fixtures(all_data: dict, output_file: Path = None):
    """Generate Python fixtures file from real responses"""
    if output_file is None:
        output_file = project_root / "tests" / "fixtures" / "real_amocrm_responses.py"
    timestamp = datetime.now().isoformat()
    content = f'''"""
 Real AMO CRM API responses from wecheap.amocrm.ru
 Generated on: {timestamp}
 """
 # Users (Пользователи) response
 USERS_RESPONSE = {json.dumps(all_data.get("users", {}), ensure_ascii=False, indent=4)}
 # Pipelines (Воронки) response
 PIPELINES_RESPONSE = {json.dumps(all_data.get("pipelines", {}), ensure_ascii=False, indent=4)}
 # Companies (Компании) response
 COMPANIES_RESPONSE = {json.dumps(all_data.get("companies", {}), ensure_ascii=False, indent=4)}
 # Contacts (Контакты) response
 CONTACTS_RESPONSE = {json.dumps(all_data.get("contacts", {}), ensure_ascii=False, indent=4)}
 # Deals (Сделки) response
 DEALS_RESPONSE = {json.dumps(all_data.get("deals", {}), ensure_ascii=False, indent=4)}
 # Events (События) response
 EVENTS_RESPONSE = {json.dumps(all_data.get("events", {}), ensure_ascii=False, indent=4)}
 # Custom fields metadata for deals
 CUSTOM_FIELDS_DEALS_RESPONSE = {json.dumps(all_data.get("custom_fields_deals", {}), ensure_ascii=False, indent=4)}
 # Custom fields metadata for contacts
 CUSTOM_FIELDS_CONTACTS_RESPONSE = {json.dumps(all_data.get("custom_fields_contacts", {}), ensure_ascii=False, indent=4)}
 # Custom fields metadata for companies
 CUSTOM_FIELDS_COMPANIES_RESPONSE = {json.dumps(all_data.get("custom_fields_companies", {}), ensure_ascii=False, indent=4)}
 '''
    with open(output_file, 'w', encoding='utf-8') as f:
        f.write(content)
    print(f"🐍 Generated Python fixtures at {output_file}")
 async def main():
    """Main function to fetch and save AMO CRM data"""
    print("🚀 Fetching AMO CRM data from wecheap.amocrm.ru")
    print(f"📡 Domain: {settings.AMO_CRM_DOMAIN}")
    if not settings.AMO_CRM_ACCESS_TOKEN:
        print("❌ Error: AMO_CRM_ACCESS_TOKEN not set!")
        print("Please set it in your .env file")
        return
    try:
        # Create client and fetch data
        client = AmoCRMClient()
        all_data = await client.fetch_all_data()
        print("\n📁 Saving responses...")
        # Save individual JSON files
        for entity_type, data in all_data.items():
            if data:  # Only save non-empty responses
                save_response_to_file(data, entity_type)
        # Generate Python fixtures
        generate_python_fixtures(all_data)
        print("\n✅ All done! Real AMO CRM responses saved for testing")
        print("\nFiles created:")
        print("- tests/fixtures/real_responses/*.json (individual JSON files)")
        print("- tests/fixtures/real_amocrm_responses.py (Python fixtures)")
        # Show summary
        print("\n📊 Data Summary:")
        for entity_type, data in all_data.items():
            if "_embedded" in data:
                embedded_key = list(data["_embedded"].keys())[0] if data["_embedded"] else "items"
                count = len(data["_embedded"].get(embedded_key, []))
                print(f"  - {entity_type}: {count} items")
            else:
                print(f"  - {entity_type}: metadata")
    except Exception as e:
        print(f"❌ Error: {e}")
        print("\nTroubleshooting:")
        print("1. Check your AMO_CRM_ACCESS_TOKEN in .env file")
        print("2. Make sure the token is valid and not expired")
        print("3. Verify you have access to wecheap.amocrm.ru")
        sys.exit(1)
 if __name__ == "__main__":
    asyncio.run(main())
--- a/servers/init.py
+++ b/servers/init.py
--- a/tests/fixtures/amocrm_responses.py
+++ b/tests/fixtures/amocrm_responses.py
@ -0,0 +1,721 @@
 """
 Real AMO CRM API response examples for testing
 Based on AMO CRM HAL+JSON format
 """
 # Users (Пользователи) response example
 USERS_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/users?page=1&limit=250"
        },
        "next": {
            "href": "https://example.amocrm.ru/api/v4/users?page=2&limit=250"
        }
    },
    "_embedded": {
        "users": [
            {
                "id": 504141,
                "name": "Иван Иванов",
                "email": "ivan@example.com",
                "lang": "ru",
                "rights": {
                    "leads": {
                        "view": "A",
                        "edit": "A",
                        "add": "A",
                        "delete": "A",
                        "export": "A"
                    },
                    "contacts": {
                        "view": "A",
                        "edit": "A",
                        "add": "A",
                        "delete": "A",
                        "export": "A"
                    }
                },
                "is_admin": True,
                "is_free": False,
                "is_active": True,
                "group_id": 0,
                "role_id": None,
                "uuid": "550e8400-e29b-41d4-a716-446655440000",
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/users/504141"
                    }
                }
            },
            {
                "id": 504142,
                "name": "Мария Петрова",
                "email": "maria@example.com",
                "lang": "ru",
                "rights": {
                    "leads": {
                        "view": "A",
                        "edit": "A",
                        "add": "A",
                        "delete": "A",
                        "export": "A"
                    }
                },
                "is_admin": False,
                "is_free": False,
                "is_active": True,
                "group_id": 0,
                "role_id": 123456,
                "uuid": "550e8400-e29b-41d4-a716-446655440001",
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/users/504142"
                    }
                }
            }
        ]
    }
 }
 # Pipelines (Воронки) response example
 PIPELINES_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/leads/pipelines"
        }
    },
    "_embedded": {
        "pipelines": [
            {
                "id": 3130966,
                "name": "Продажи",
                "sort": 1,
                "is_main": True,
                "is_unsorted": False,
                "is_archive": False,
                "account_id": 28805383,
                "created_at": 1640995200,
                "updated_at": 1704067200,
                "_embedded": {
                    "statuses": [
                        {
                            "id": 32532070,
                            "name": "Неразобранное",
                            "sort": 10,
                            "is_editable": False,
                            "pipeline_id": 3130966,
                            "color": "#c1c1c1",
                            "type": 1,
                            "account_id": 28805383,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/leads/pipelines/3130966/statuses/32532070"
                                }
                            }
                        },
                        {
                            "id": 32532073,
                            "name": "Первичный контакт",
                            "sort": 20,
                            "is_editable": True,
                            "pipeline_id": 3130966,
                            "color": "#99ccff",
                            "type": 0,
                            "account_id": 28805383,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/leads/pipelines/3130966/statuses/32532073"
                                }
                            }
                        },
                        {
                            "id": 32532076,
                            "name": "Переговоры",
                            "sort": 30,
                            "is_editable": True,
                            "pipeline_id": 3130966,
                            "color": "#ffff99",
                            "type": 0,
                            "account_id": 28805383,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/leads/pipelines/3130966/statuses/32532076"
                                }
                            }
                        },
                        {
                            "id": 142,
                            "name": "Успешно реализовано",
                            "sort": 10000,
                            "is_editable": False,
                            "pipeline_id": 3130966,
                            "color": "#CCFF66",
                            "type": 0,
                            "account_id": 28805383,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/leads/pipelines/3130966/statuses/142"
                                }
                            }
                        },
                        {
                            "id": 143,
                            "name": "Закрыто и не реализовано",
                            "sort": 11000,
                            "is_editable": False,
                            "pipeline_id": 3130966,
                            "color": "#D5D8DB",
                            "type": 0,
                            "account_id": 28805383,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/leads/pipelines/3130966/statuses/143"
                                }
                            }
                        }
                    ]
                },
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/pipelines/3130966"
                    }
                }
            }
        ]
    }
 }
 # Companies (Компании) response example
 COMPANIES_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/companies?page=1&limit=250"
        },
        "next": {
            "href": "https://example.amocrm.ru/api/v4/companies?page=2&limit=250"
        }
    },
    "_embedded": {
        "companies": [
            {
                "id": 15960673,
                "name": "ООО \"Рога и копыта\"",
                "responsible_user_id": 504141,
                "group_id": 0,
                "created_by": 504141,
                "updated_by": 504142,
                "created_at": 1640995200,
                "updated_at": 1704067200,
                "closest_task_at": None,
                "is_deleted": False,
                "custom_fields_values": [
                    {
                        "field_id": 123456,
                        "field_name": "Телефон",
                        "field_code": "PHONE",
                        "field_type": "multitext",
                        "values": [
                            {
                                "value": "+7 (495) 123-45-67",
                                "enum_id": 456789,
                                "enum": "WORK"
                            }
                        ]
                    },
                    {
                        "field_id": 123457,
                        "field_name": "Email",
                        "field_code": "EMAIL",
                        "field_type": "multitext",
                        "values": [
                            {
                                "value": "info@example.com",
                                "enum_id": 456790,
                                "enum": "WORK"
                            }
                        ]
                    },
                    {
                        "field_id": 987654,
                        "field_name": "Отрасль",
                        "field_code": None,
                        "field_type": "select",
                        "values": [
                            {
                                "value": "IT",
                                "enum_id": 111222,
                                "enum": "IT"
                            }
                        ]
                    }
                ],
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/companies/15960673"
                    }
                }
            }
        ]
    }
 }
 # Contacts (Контакты) response example
 CONTACTS_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/contacts?page=1&limit=250"
        },
        "next": {
            "href": "https://example.amocrm.ru/api/v4/contacts?page=2&limit=250"
        }
    },
    "_embedded": {
        "contacts": [
            {
                "id": 19421421,
                "name": "Алексей Смирнов",
                "first_name": "Алексей",
                "last_name": "Смирнов",
                "responsible_user_id": 504141,
                "group_id": 0,
                "created_by": 504141,
                "updated_by": 504141,
                "created_at": 1640995200,
                "updated_at": 1704067200,
                "closest_task_at": 1704153600,
                "is_deleted": False,
                "custom_fields_values": [
                    {
                        "field_id": 123456,
                        "field_name": "Телефон",
                        "field_code": "PHONE",
                        "field_type": "multitext",
                        "values": [
                            {
                                "value": "+7 (903) 123-45-67",
                                "enum_id": 456789,
                                "enum": "MOB"
                            }
                        ]
                    },
                    {
                        "field_id": 123457,
                        "field_name": "Email",
                        "field_code": "EMAIL",
                        "field_type": "multitext",
                        "values": [
                            {
                                "value": "alexey@example.com",
                                "enum_id": 456790,
                                "enum": "PRIV"
                            }
                        ]
                    },
                    {
                        "field_id": 555666,
                        "field_name": "Должность",
                        "field_code": None,
                        "field_type": "text",
                        "values": [
                            {
                                "value": "Менеджер по продажам"
                            }
                        ]
                    }
                ],
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/contacts/19421421"
                    }
                },
                "_embedded": {
                    "companies": [
                        {
                            "id": 15960673,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/companies/15960673"
                                }
                            }
                        }
                    ]
                }
            }
        ]
    }
 }
 # Deals (Сделки) response example
 DEALS_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/leads?page=1&limit=250"
        },
        "next": {
            "href": "https://example.amocrm.ru/api/v4/leads?page=2&limit=250"
        }
    },
    "_embedded": {
        "leads": [
            {
                "id": 19620805,
                "name": "Сделка с ООО \"Рога и копыта\"",
                "price": 150000,
                "responsible_user_id": 504141,
                "group_id": 0,
                "status_id": 32532073,
                "pipeline_id": 3130966,
                "loss_reason_id": None,
                "created_by": 504141,
                "updated_by": 504141,
                "closed_at": None,
                "created_at": 1640995200,
                "updated_at": 1704067200,
                "closest_task_at": 1704153600,
                "is_deleted": False,
                "custom_fields_values": [
                    {
                        "field_id": 777888,
                        "field_name": "Приоритет",
                        "field_code": None,
                        "field_type": "select",
                        "values": [
                            {
                                "value": "Высокий",
                                "enum_id": 999111,
                                "enum": "HIGH"
                            }
                        ]
                    },
                    {
                        "field_id": 777889,
                        "field_name": "Источник",
                        "field_code": None,
                        "field_type": "text",
                        "values": [
                            {
                                "value": "Холодный звонок"
                            }
                        ]
                    },
                    {
                        "field_id": 777890,
                        "field_name": "Дата закрытия",
                        "field_code": None,
                        "field_type": "date",
                        "values": [
                            {
                                "value": "1704326400"
                            }
                        ]
                    },
                    {
                        "field_id": 777891,
                        "field_name": "Бюджет клиента",
                        "field_code": None,
                        "field_type": "numeric",
                        "values": [
                            {
                                "value": "200000"
                            }
                        ]
                    }
                ],
                "score": None,
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/19620805"
                    }
                },
                "_embedded": {
                    "contacts": [
                        {
                            "id": 19421421,
                            "is_main": True,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/contacts/19421421"
                                }
                            }
                        }
                    ],
                    "companies": [
                        {
                            "id": 15960673,
                            "is_main": True,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/companies/15960673"
                                }
                            }
                        }
                    ]
                }
            },
            {
                "id": 19620806,
                "name": "Повторная сделка",
                "price": 75000,
                "responsible_user_id": 504142,
                "group_id": 0,
                "status_id": 32532076,
                "pipeline_id": 3130966,
                "loss_reason_id": None,
                "created_by": 504142,
                "updated_by": 504142,
                "closed_at": None,
                "created_at": 1701388800,
                "updated_at": 1704067200,
                "closest_task_at": None,
                "is_deleted": False,
                "custom_fields_values": [
                    {
                        "field_id": 777888,
                        "field_name": "Приоритет",
                        "field_code": None,
                        "field_type": "select",
                        "values": [
                            {
                                "value": "Средний",
                                "enum_id": 999112,
                                "enum": "MEDIUM"
                            }
                        ]
                    }
                ],
                "score": None,
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/19620806"
                    }
                },
                "_embedded": {
                    "contacts": [
                        {
                            "id": 19421421,
                            "is_main": False,
                            "_links": {
                                "self": {
                                    "href": "https://example.amocrm.ru/api/v4/contacts/19421421"
                                }
                            }
                        }
                    ]
                }
            }
        ]
    }
 }
 # Events (События) response example
 EVENTS_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/events?page=1&limit=250"
        },
        "next": {
            "href": "https://example.amocrm.ru/api/v4/events?page=2&limit=250"
        }
    },
    "_embedded": {
        "events": [
            {
                "id": "550e8400-e29b-41d4-a716-446655440000",
                "type": "lead_status_changed",
                "entity_id": 19620805,
                "entity_type": "lead",
                "created_by": 504141,
                "created_at": 1704067200,
                "value_after": [
                    {
                        "lead_status": {
                            "id": 32532073,
                            "name": "Первичный контакт",
                            "color": "#99ccff",
                            "pipeline_id": 3130966
                        }
                    }
                ],
                "value_before": [
                    {
                        "lead_status": {
                            "id": 32532070,
                            "name": "Неразобранное",
                            "color": "#c1c1c1",
                            "pipeline_id": 3130966
                        }
                    }
                ],
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/events/550e8400-e29b-41d4-a716-446655440000"
                    }
                }
            },
            {
                "id": "550e8400-e29b-41d4-a716-446655440001",
                "type": "incoming_call",
                "entity_id": 19421421,
                "entity_type": "contact",
                "created_by": 504141,
                "created_at": 1704060000,
                "value_after": [
                    {
                        "call": {
                            "duration": 120,
                            "phone": "+7 (903) 123-45-67",
                            "call_status": "Дозвонился",
                            "call_result": "Договорились о встрече"
                        }
                    }
                ],
                "value_before": [],
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/events/550e8400-e29b-41d4-a716-446655440001"
                    }
                }
            },
            {
                "id": "550e8400-e29b-41d4-a716-446655440002",
                "type": "outgoing_call",
                "entity_id": 15960673,
                "entity_type": "company",
                "created_by": 504142,
                "created_at": 1704052800,
                "value_after": [
                    {
                        "call": {
                            "duration": 0,
                            "phone": "+7 (495) 123-45-67",
                            "call_status": "Не дозвонился",
                            "call_result": "Занято"
                        }
                    }
                ],
                "value_before": [],
                "account_id": 28805383,
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/events/550e8400-e29b-41d4-a716-446655440002"
                    }
                }
            }
        ]
    }
 }
 # Custom fields metadata response (for understanding field types)
 CUSTOM_FIELDS_RESPONSE = {
    "_page": 1,
    "_links": {
        "self": {
            "href": "https://example.amocrm.ru/api/v4/leads/custom_fields"
        }
    },
    "_embedded": {
        "custom_fields": [
            {
                "id": 777888,
                "name": "Приоритет",
                "type": "select",
                "account_id": 28805383,
                "code": None,
                "sort": 1,
                "api_code": None,
                "is_computed": False,
                "is_predefined": False,
                "entity_type": "leads",
                "enums": [
                    {
                        "id": 999111,
                        "value": "Высокий",
                        "sort": 1,
                        "api_code": "HIGH"
                    },
                    {
                        "id": 999112,
                        "value": "Средний",
                        "sort": 2,
                        "api_code": "MEDIUM"
                    },
                    {
                        "id": 999113,
                        "value": "Низкий",
                        "sort": 3,
                        "api_code": "LOW"
                    }
                ],
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/custom_fields/777888"
                    }
                }
            },
            {
                "id": 777889,
                "name": "Источник",
                "type": "text",
                "account_id": 28805383,
                "code": None,
                "sort": 2,
                "api_code": None,
                "is_computed": False,
                "is_predefined": False,
                "entity_type": "leads",
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/custom_fields/777889"
                    }
                }
            },
            {
                "id": 777890,
                "name": "Дата закрытия",
                "type": "date",
                "account_id": 28805383,
                "code": None,
                "sort": 3,
                "api_code": None,
                "is_computed": False,
                "is_predefined": False,
                "entity_type": "leads",
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/custom_fields/777890"
                    }
                }
            },
            {
                "id": 777891,
                "name": "Бюджет клиента",
                "type": "numeric",
                "account_id": 28805383,
                "code": None,
                "sort": 4,
                "api_code": None,
                "is_computed": False,
                "is_predefined": False,
                "entity_type": "leads",
                "_links": {
                    "self": {
                        "href": "https://example.amocrm.ru/api/v4/leads/custom_fields/777891"
                    }
                }
            }
        ]
    }
 }
--- a/tests/test_amocrm_examples.py
+++ b/tests/test_amocrm_examples.py
@ -0,0 +1,197 @@
 """
 Test file demonstrating AMO CRM response examples
 """
 import pytest
 from tests.fixtures.amocrm_responses import (
    USERS_RESPONSE,
    PIPELINES_RESPONSE,
    COMPANIES_RESPONSE,
    CONTACTS_RESPONSE,
    DEALS_RESPONSE,
    EVENTS_RESPONSE,
    CUSTOM_FIELDS_RESPONSE
 )
 def test_users_response_structure():
    """Test that users response has expected structure"""
    assert "_embedded" in USERS_RESPONSE
    assert "users" in USERS_RESPONSE["_embedded"]
    users = USERS_RESPONSE["_embedded"]["users"]
    assert len(users) == 2
    first_user = users[0]
    assert first_user["id"] == 504141
    assert first_user["name"] == "Иван Иванов"
    assert first_user["email"] == "ivan@example.com"
    assert first_user["is_active"] is True
 def test_pipelines_response_structure():
    """Test that pipelines response has expected structure"""
    assert "_embedded" in PIPELINES_RESPONSE
    assert "pipelines" in PIPELINES_RESPONSE["_embedded"]
    pipelines = PIPELINES_RESPONSE["_embedded"]["pipelines"]
    assert len(pipelines) == 1
    pipeline = pipelines[0]
    assert pipeline["id"] == 3130966
    assert pipeline["name"] == "Продажи"
    assert pipeline["is_main"] is True
    # Check stages
    assert "_embedded" in pipeline
    assert "statuses" in pipeline["_embedded"]
    statuses = pipeline["_embedded"]["statuses"]
    assert len(statuses) == 5
 def test_companies_response_structure():
    """Test that companies response has expected structure"""
    assert "_embedded" in COMPANIES_RESPONSE
    assert "companies" in COMPANIES_RESPONSE["_embedded"]
    companies = COMPANIES_RESPONSE["_embedded"]["companies"]
    assert len(companies) == 1
    company = companies[0]
    assert company["id"] == 15960673
    assert company["name"] == "ООО \"Рога и копыта\""
    assert company["responsible_user_id"] == 504141
    # Check custom fields
    assert "custom_fields_values" in company
    custom_fields = company["custom_fields_values"]
    assert len(custom_fields) == 3
    # Check phone field
    phone_field = next(f for f in custom_fields if f["field_name"] == "Телефон")
    assert phone_field["field_type"] == "multitext"
    assert phone_field["values"][0]["value"] == "+7 (495) 123-45-67"
 def test_contacts_response_structure():
    """Test that contacts response has expected structure"""
    assert "_embedded" in CONTACTS_RESPONSE
    assert "contacts" in CONTACTS_RESPONSE["_embedded"]
    contacts = CONTACTS_RESPONSE["_embedded"]["contacts"]
    assert len(contacts) == 1
    contact = contacts[0]
    assert contact["id"] == 19421421
    assert contact["name"] == "Алексей Смирнов"
    assert contact["first_name"] == "Алексей"
    assert contact["last_name"] == "Смирнов"
    # Check embedded companies
    assert "_embedded" in contact
    assert "companies" in contact["_embedded"]
    companies = contact["_embedded"]["companies"]
    assert len(companies) == 1
    assert companies[0]["id"] == 15960673
 def test_deals_response_structure():
    """Test that deals response has expected structure"""
    assert "_embedded" in DEALS_RESPONSE
    assert "leads" in DEALS_RESPONSE["_embedded"]  # Note: AMO CRM calls deals "leads"
    deals = DEALS_RESPONSE["_embedded"]["leads"]
    assert len(deals) == 2
    first_deal = deals[0]
    assert first_deal["id"] == 19620805
    assert first_deal["name"] == "Сделка с ООО \"Рога и копыта\""
    assert first_deal["price"] == 150000
    assert first_deal["status_id"] == 32532073
    assert first_deal["pipeline_id"] == 3130966
    # Check custom fields
    custom_fields = first_deal["custom_fields_values"]
    priority_field = next(f for f in custom_fields if f["field_name"] == "Приоритет")
    assert priority_field["field_type"] == "select"
    assert priority_field["values"][0]["value"] == "Высокий"
    # Check embedded relationships
    assert "_embedded" in first_deal
    assert "contacts" in first_deal["_embedded"]
    assert "companies" in first_deal["_embedded"]
    contacts = first_deal["_embedded"]["contacts"]
    assert contacts[0]["id"] == 19421421
    assert contacts[0]["is_main"] is True
 def test_events_response_structure():
    """Test that events response has expected structure"""
    assert "_embedded" in EVENTS_RESPONSE
    assert "events" in EVENTS_RESPONSE["_embedded"]
    events = EVENTS_RESPONSE["_embedded"]["events"]
    assert len(events) == 3
    # Test lead status changed event
    status_event = next(e for e in events if e["type"] == "lead_status_changed")
    assert status_event["entity_type"] == "lead"
    assert status_event["entity_id"] == 19620805
    assert "value_after" in status_event
    assert "value_before" in status_event
    # Test incoming call event
    call_event = next(e for e in events if e["type"] == "incoming_call")
    assert call_event["entity_type"] == "contact"
    assert call_event["entity_id"] == 19421421
    assert call_event["value_after"][0]["call"]["duration"] == 120
 def test_custom_fields_metadata():
    """Test custom fields metadata structure"""
    assert "_embedded" in CUSTOM_FIELDS_RESPONSE
    assert "custom_fields" in CUSTOM_FIELDS_RESPONSE["_embedded"]
    fields = CUSTOM_FIELDS_RESPONSE["_embedded"]["custom_fields"]
    assert len(fields) == 4
    # Test select field with enums
    priority_field = next(f for f in fields if f["name"] == "Приоритет")
    assert priority_field["type"] == "select"
    assert "enums" in priority_field
    assert len(priority_field["enums"]) == 3
    high_priority = priority_field["enums"][0]
    assert high_priority["value"] == "Высокий"
    assert high_priority["api_code"] == "HIGH"
    # Test date field
    date_field = next(f for f in fields if f["name"] == "Дата закрытия")
    assert date_field["type"] == "date"
    # Test numeric field
    numeric_field = next(f for f in fields if f["name"] == "Бюджет клиента")
    assert numeric_field["type"] == "numeric"
 def test_date_validation():
    """Test that dates are in expected range (2017-2026)"""
    # Test deals dates
    deals = DEALS_RESPONSE["_embedded"]["leads"]
    for deal in deals:
        created_at = deal["created_at"]
        updated_at = deal["updated_at"]
        # Check dates are in range (2017 = 1483228800, 2026 = 1767225600)
        assert 1483228800 <= created_at <= 1767225600, f"created_at {created_at} out of range"
        assert 1483228800 <= updated_at <= 1767225600, f"updated_at {updated_at} out of range"
    # Test events dates
    events = EVENTS_RESPONSE["_embedded"]["events"]
    for event in events:
        created_at = event["created_at"]
        assert 1483228800 <= created_at <= 1767225600, f"event created_at {created_at} out of range"
 if __name__ == "__main__":
    pytest.main([__file__])
--- a/tests/test_api.py
+++ b/tests/test_api.py
@ -0,0 +1,253 @@
 """
 Test API endpoints with AMO CRM data examples
 """
 import pytest
 from fastapi.testclient import TestClient
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.pool import StaticPool
 from app import app
 from adapters.sqlite.database import Base, get_db
 from tests.fixtures.amocrm_responses import USERS_RESPONSE, DEALS_RESPONSE
 # Create test database
 SQLALCHEMY_DATABASE_URL = "sqlite:///./test.db"
 engine = create_engine(
    SQLALCHEMY_DATABASE_URL,
    connect_args={"check_same_thread": False},
    poolclass=StaticPool,
 )
 TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 # Create tables
 Base.metadata.create_all(bind=engine)
 def override_get_db():
    try:
        db = TestingSessionLocal()
        yield db
    finally:
        db.close()
 app.dependency_overrides[get_db] = override_get_db
 client = TestClient(app)
 def test_root_endpoint():
    """Test root endpoint"""
    response = client.get("/")
    assert response.status_code == 200
    data = response.json()
    assert data["message"] == "AMO CRM Data Collection Service"
    assert data["version"] == "0.1.0"
 def test_health_endpoint():
    """Test health check endpoint"""
    response = client.get("/health")
    assert response.status_code == 200
    data = response.json()
    assert data["status"] == "healthy"
 def test_list_entities_empty():
    """Test listing entities when database is empty"""
    response = client.get("/api/v1/entities/")
    assert response.status_code == 200
    data = response.json()
    assert "entities" in data
    # All counts should be 0
    for entity_type in ["deals", "contacts", "companies", "pipelines", "users", "events"]:
        assert entity_type in data["entities"]
        assert data["entities"][entity_type]["count"] == 0
 def test_put_users_data():
    """Test putting users data"""
    users_data = USERS_RESPONSE["_embedded"]["users"]
    response = client.post(
        "/api/v1/data/users",
        json={
            "data": users_data,
            "sync_mode": "upsert"
        }
    )
    assert response.status_code == 200
    data = response.json()
    assert data["processed_count"] == 2
    assert "Processed 2 users" in data["message"]
 def test_put_deals_data():
    """Test putting deals data"""
    deals_data = DEALS_RESPONSE["_embedded"]["leads"]
    response = client.post(
        "/api/v1/data/deals",
        json={
            "data": deals_data,
            "sync_mode": "upsert"
        }
    )
    assert response.status_code == 200
    data = response.json()
    assert data["processed_count"] == 2
    assert "Processed 2 deals" in data["message"]
 def test_list_entities_with_data():
    """Test listing entities after adding data"""
    response = client.get("/api/v1/entities/")
    assert response.status_code == 200
    data = response.json()
    # Should have users and deals now
    assert data["entities"]["users"]["count"] == 2
    assert data["entities"]["deals"]["count"] == 2
 def test_list_deal_fields():
    """Test listing deal fields"""
    response = client.get("/api/v1/entities/deals/fields")
    assert response.status_code == 200
    data = response.json()
    assert data["entity_type"] == "deals"
    assert "fields" in data
    # Should have standard fields
    field_names = [f["name"] for f in data["fields"]]
    assert "name" in field_names
    assert "price" in field_names
    assert "created_at" in field_names
    # Should have custom fields from the test data
    custom_fields = [f for f in data["fields"] if f["is_custom"]]
    assert len(custom_fields) > 0
 def test_export_configuration():
    """Test creating export configuration"""
    config_data = {
        "name": "Test Export Configuration",
        "sheet_id": "test-sheet-id-123",
        "date_range_start": "2024-01-01T00:00:00Z",
        "date_range_end": "2024-12-31T23:59:59Z",
        "entity_mappings": {
            "deals": {
                "sheet_name": "Deals",
                "is_enabled": True,
                "field_mapping": [
                    {"field_name": "name", "column": "A", "order": 1},
                    {"field_name": "price", "column": "B", "order": 2}
                ]
            },
            "users": {
                "sheet_name": "Users",
                "is_enabled": True,
                "field_mapping": [
                    {"field_name": "name", "column": "A", "order": 1},
                    {"field_name": "email", "column": "B", "order": 2}
                ]
            }
        }
    }
    response = client.post("/api/v1/export/configure", json=config_data)
    assert response.status_code == 200
    data = response.json()
    assert "configuration_id" in data
    assert data["message"] == "Export configuration created successfully"
    return data["configuration_id"]
 def test_list_export_configurations():
    """Test listing export configurations"""
    # First create a configuration
    config_id = test_export_configuration()
    response = client.get("/api/v1/export/configurations")
    assert response.status_code == 200
    data = response.json()
    assert "configurations" in data
    assert len(data["configurations"]) >= 1
    config = data["configurations"][0]
    assert config["name"] == "Test Export Configuration"
    assert config["sheet_id"] == "test-sheet-id-123"
    assert "entity_mappings" in config
 def test_start_export_job():
    """Test starting export job"""
    # First create a configuration
    config_id = test_export_configuration()
    response = client.post(
        "/api/v1/export/start",
        json={"configuration_id": config_id}
    )
    assert response.status_code == 200
    data = response.json()
    assert "job_id" in data
    assert data["status"] == "pending"
    assert data["message"] == "Export job queued successfully"
    return data["job_id"]
 def test_export_job_status():
    """Test getting export job status"""
    # First start a job
    job_id = test_start_export_job()
    response = client.get(f"/api/v1/export/status/{job_id}")
    assert response.status_code == 200
    data = response.json()
    assert data["job_id"] == job_id
    assert data["status"] == "pending"
    assert "progress" in data
    assert data["progress"]["records_processed"] == 0
 def test_list_export_jobs():
    """Test listing export jobs"""
    # First start a job
    job_id = test_start_export_job()
    response = client.get("/api/v1/export/jobs")
    assert response.status_code == 200
    data = response.json()
    assert "jobs" in data
    assert data["total"] >= 1
    assert data["page"] == 1
    assert data["per_page"] == 20
    # Should contain our job
    job_ids = [job["job_id"] for job in data["jobs"]]
    assert job_id in job_ids
 def test_invalid_entity_type():
    """Test invalid entity type"""
    response = client.get("/api/v1/entities/invalid_entity/fields")
    assert response.status_code == 404
    data = response.json()
    assert "Entity type not found" in data["detail"]
 if __name__ == "__main__":
    pytest.main([__file__])
--- a/utils/init.py
+++ b/utils/init.py
@ -0,0 +1 @@
 # Utils package
--- a/utils/config.py
+++ b/utils/config.py
@ -0,0 +1,31 @@
 from pydantic_settings import BaseSettings
 from typing import Optional
 class Settings(BaseSettings):
    # Database
    DATABASE_URL: str = "sqlite:///./amo_data.db"
    # AMO CRM API
    AMO_CRM_DOMAIN: str = "wecheap.amocrm.ru"
    AMO_CRM_ACCESS_TOKEN: str
    # Google Sheets API
    GOOGLE_SERVICE_ACCOUNT_FILE: str
    GOOGLE_SCOPES: str = "https://www.googleapis.com/auth/spreadsheets"
    # Redis (for Celery)
    REDIS_URL: str = "redis://localhost:6379/0"
    # API Settings
    API_V1_STR: str = "/api/v1"
    # Logging
    LOG_LEVEL: str = "INFO"
    class Config:
        env_file = ".env"
        case_sensitive = True
 settings = Settings()
--- a/uv.lock
+++ b/uv.lock
--- a/workers/init.py
+++ b/workers/init.py