Maxim Snesarev
33d6bb7ebd
- Updated database configuration to switch from SQLite to PostgreSQL, including changes to alembic.ini, Docker Compose, and environment settings. - Refactored application code to utilize PostgreSQL database adapters, ensuring compatibility with the new database structure. - Enhanced API routes and data handling to support the new database, including adjustments in data models and query logic. - Introduced new job processing mechanisms for full synchronization of AMO CRM entities, leveraging FastStream for background tasks. - Improved logging and error handling across the application to facilitate better monitoring and debugging. - Removed obsolete SQLite adapter files and migrations, streamlining the project structure for PostgreSQL integration.
258 lines
8.6 KiB
Python
258 lines
8.6 KiB
Python
"""
|
|
Sync server for processing AMO CRM data synchronization jobs.
|
|
|
|
This module handles synchronization of data from AMO CRM API to the local
|
|
SQLite database, including both full syncs and incremental updates.
|
|
"""
|
|
|
|
import logging
|
|
import httpx
|
|
from typing import Dict, Any, List, Optional
|
|
from datetime import datetime, timezone
|
|
from adapters.amocrm_client import AmoCRMClient
|
|
from adapters.postgres.database import SessionLocal
|
|
from utils.config import settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class SyncServer:
|
|
"""Server for processing AMO CRM data synchronization operations."""
|
|
|
|
def __init__(self):
|
|
"""Initialize SyncServer with AMO CRM client and database."""
|
|
self.amocrm = AmoCRMClient()
|
|
self.base_data_url = f"{settings.API_BASE_URL}{settings.API_V1_STR}/data"
|
|
|
|
async def process_sync_job(self, sync_data: Dict[str, Any]) -> None:
|
|
"""
|
|
Process AMO CRM data synchronization job.
|
|
|
|
Args:
|
|
sync_data: Dictionary containing job_id, entity_type, and parameters
|
|
"""
|
|
job_id = sync_data["job_id"]
|
|
entity_type = sync_data["entity_type"]
|
|
parameters = sync_data.get("parameters", {})
|
|
|
|
try:
|
|
logger.info(f"Starting sync job {job_id} for {entity_type}")
|
|
|
|
# Validate entity type
|
|
if not self._is_valid_entity_type(entity_type):
|
|
raise ValueError(f"Invalid entity type: {entity_type}")
|
|
|
|
# Fetch data from AMO CRM
|
|
data = await self.amocrm.fetch_entity_data(entity_type, **parameters)
|
|
|
|
if not data:
|
|
logger.warning(f"No data received from AMO CRM for {entity_type}")
|
|
return
|
|
|
|
# Process and store data
|
|
processed_count = await self._store_entity_data(entity_type, data)
|
|
|
|
logger.info(f"Sync job {job_id} completed: {processed_count} records processed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Sync job {job_id} failed: {str(e)}")
|
|
raise
|
|
|
|
async def refresh_entity_data(self, entity_type: str) -> None:
|
|
"""
|
|
Refresh all data for an entity type with incremental updates.
|
|
|
|
Args:
|
|
entity_type: Type of entity to refresh (deals, contacts, companies, etc.)
|
|
"""
|
|
try:
|
|
logger.info(f"Starting refresh for {entity_type} data")
|
|
|
|
# Validate entity type
|
|
if not self._is_valid_entity_type(entity_type):
|
|
raise ValueError(f"Invalid entity type: {entity_type}")
|
|
|
|
# Get last update timestamp for incremental sync
|
|
last_update = await self._get_last_update_timestamp(entity_type)
|
|
|
|
# Fetch updated data from AMO CRM
|
|
data = await self.amocrm.fetch_entity_data(
|
|
entity_type,
|
|
updated_at=last_update,
|
|
limit=250 # Process in batches
|
|
)
|
|
|
|
if data:
|
|
processed_count = await self._store_entity_data(entity_type, data)
|
|
logger.info(f"Refresh completed for {entity_type}: {processed_count} records updated")
|
|
else:
|
|
logger.info(f"No updates found for {entity_type}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Refresh failed for {entity_type}: {str(e)}")
|
|
raise
|
|
|
|
async def full_sync_entity(self, entity_type: str, batch_size: int = 250) -> int:
|
|
"""
|
|
Perform a full synchronization of an entity type.
|
|
|
|
Args:
|
|
entity_type: Type of entity to sync
|
|
batch_size: Number of records to fetch per batch
|
|
|
|
Returns:
|
|
Total number of records processed
|
|
"""
|
|
try:
|
|
logger.info(f"Starting full sync for {entity_type}")
|
|
|
|
total_processed = 0
|
|
page = 1
|
|
|
|
while True:
|
|
# Fetch batch of data
|
|
data = await self.amocrm.fetch_entity_data(
|
|
entity_type,
|
|
limit=batch_size,
|
|
page=page
|
|
)
|
|
|
|
if not data:
|
|
break
|
|
|
|
# Process batch
|
|
batch_processed = await self._store_entity_data(entity_type, data)
|
|
total_processed += batch_processed
|
|
|
|
logger.info(f"Processed batch {page}: {batch_processed} {entity_type} records")
|
|
|
|
# Check if we got less than batch_size (last page)
|
|
if len(data) < batch_size:
|
|
break
|
|
|
|
page += 1
|
|
|
|
logger.info(f"Full sync completed for {entity_type}: {total_processed} total records")
|
|
return total_processed
|
|
|
|
except Exception as e:
|
|
logger.error(f"Full sync failed for {entity_type}: {str(e)}")
|
|
raise
|
|
|
|
async def _store_entity_data(self, entity_type: str, data: List[Dict[str, Any]]) -> int:
|
|
"""
|
|
Store entity data in the database by calling data ingestion endpoints.
|
|
|
|
Args:
|
|
entity_type: Type of entity being stored
|
|
data: List of entity records from AMO CRM
|
|
|
|
Returns:
|
|
Number of records processed
|
|
"""
|
|
try:
|
|
if not data:
|
|
return 0
|
|
|
|
# Call the data ingestion API endpoint
|
|
endpoint = f"{self.base_data_url}/{entity_type}"
|
|
|
|
payload = {
|
|
"data": data,
|
|
"sync_mode": "upsert" # Update existing, insert new
|
|
}
|
|
|
|
async with httpx.AsyncClient(timeout=300.0) as client:
|
|
response = await client.post(endpoint, json=payload)
|
|
response.raise_for_status()
|
|
|
|
result = response.json()
|
|
processed_count = result.get("processed_count", 0)
|
|
|
|
# Update last sync timestamp
|
|
await self._update_last_sync_timestamp(entity_type)
|
|
|
|
logger.info(f"Stored {processed_count} {entity_type} records in database")
|
|
return processed_count
|
|
|
|
except httpx.HTTPError as e:
|
|
logger.error(f"HTTP error storing {entity_type} data: {str(e)}")
|
|
raise
|
|
except Exception as e:
|
|
logger.error(f"Failed to store {entity_type} data: {str(e)}")
|
|
raise
|
|
|
|
async def _get_last_update_timestamp(self, entity_type: str) -> Optional[int]:
|
|
"""
|
|
Get the timestamp of the last successful sync for an entity type.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
|
|
Returns:
|
|
Unix timestamp of last update or None for full sync
|
|
"""
|
|
try:
|
|
from adapters.postgres.models import Deal, Contact, Company, User, Pipeline, Event
|
|
|
|
# Map entity types to models
|
|
model_map = {
|
|
"deals": Deal,
|
|
"contacts": Contact,
|
|
"companies": Company,
|
|
"users": User,
|
|
"pipelines": Pipeline,
|
|
"events": Event
|
|
}
|
|
|
|
model = model_map.get(entity_type)
|
|
if not model:
|
|
logger.warning(f"Unknown entity type: {entity_type}")
|
|
return None
|
|
|
|
# Query for the latest updated_at timestamp
|
|
db = SessionLocal()
|
|
try:
|
|
result = db.query(model).order_by(model.updated_at.desc()).first()
|
|
if result and result.updated_at:
|
|
logger.debug(f"Last update timestamp for {entity_type}: {result.updated_at}")
|
|
return result.updated_at
|
|
else:
|
|
logger.debug(f"No previous records found for {entity_type}, performing full sync")
|
|
return None
|
|
finally:
|
|
db.close()
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error getting last update timestamp for {entity_type}: {str(e)}")
|
|
return None # Fall back to full sync
|
|
|
|
async def _update_last_sync_timestamp(self, entity_type: str) -> None:
|
|
"""
|
|
Update the last sync timestamp for an entity type.
|
|
|
|
Note: The actual updated_at timestamps are stored in each entity record
|
|
by the data ingestion endpoints, so this method is primarily for logging.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
"""
|
|
current_time = datetime.now(timezone.utc)
|
|
logger.debug(f"Sync completed for {entity_type} at {current_time}")
|
|
|
|
def _is_valid_entity_type(self, entity_type: str) -> bool:
|
|
"""
|
|
Validate if the entity type is supported.
|
|
|
|
Args:
|
|
entity_type: Type of entity to validate
|
|
|
|
Returns:
|
|
True if valid, False otherwise
|
|
"""
|
|
valid_types = {
|
|
"deals", "contacts", "companies",
|
|
"users", "pipelines", "events"
|
|
}
|
|
return entity_type in valid_types
|