""" Sync server for processing AMO CRM data synchronization jobs. This module handles synchronization of data from AMO CRM API to the local SQLite database, including both full syncs and incremental updates. """ import logging from typing import Dict, Any, List, Optional from datetime import datetime, timezone from adapters.amocrm_client import AmoCRMClient from adapters.sqlite.database import get_db logger = logging.getLogger(__name__) class SyncServer: """Server for processing AMO CRM data synchronization operations.""" def __init__(self): """Initialize SyncServer with AMO CRM client and database.""" self.amocrm = AmoCRMClient() self.db = get_db async def process_sync_job(self, sync_data: Dict[str, Any]) -> None: """ Process AMO CRM data synchronization job. Args: sync_data: Dictionary containing job_id, entity_type, and parameters """ job_id = sync_data["job_id"] entity_type = sync_data["entity_type"] parameters = sync_data.get("parameters", {}) try: logger.info(f"Starting sync job {job_id} for {entity_type}") # Validate entity type if not self._is_valid_entity_type(entity_type): raise ValueError(f"Invalid entity type: {entity_type}") # Fetch data from AMO CRM data = await self.amocrm.fetch_entity_data(entity_type, **parameters) if not data: logger.warning(f"No data received from AMO CRM for {entity_type}") return # Process and store data processed_count = await self._store_entity_data(entity_type, data) logger.info(f"Sync job {job_id} completed: {processed_count} records processed") except Exception as e: logger.error(f"Sync job {job_id} failed: {str(e)}") raise async def refresh_entity_data(self, entity_type: str) -> None: """ Refresh all data for an entity type with incremental updates. Args: entity_type: Type of entity to refresh (deals, contacts, companies, etc.) """ try: logger.info(f"Starting refresh for {entity_type} data") # Validate entity type if not self._is_valid_entity_type(entity_type): raise ValueError(f"Invalid entity type: {entity_type}") # Get last update timestamp for incremental sync last_update = await self._get_last_update_timestamp(entity_type) # Fetch updated data from AMO CRM data = await self.amocrm.fetch_entity_data( entity_type, updated_at=last_update, limit=250 # Process in batches ) if data: processed_count = await self._store_entity_data(entity_type, data) logger.info(f"Refresh completed for {entity_type}: {processed_count} records updated") else: logger.info(f"No updates found for {entity_type}") except Exception as e: logger.error(f"Refresh failed for {entity_type}: {str(e)}") raise async def full_sync_entity(self, entity_type: str, batch_size: int = 250) -> int: """ Perform a full synchronization of an entity type. Args: entity_type: Type of entity to sync batch_size: Number of records to fetch per batch Returns: Total number of records processed """ try: logger.info(f"Starting full sync for {entity_type}") total_processed = 0 page = 1 while True: # Fetch batch of data data = await self.amocrm.fetch_entity_data( entity_type, limit=batch_size, page=page ) if not data: break # Process batch batch_processed = await self._store_entity_data(entity_type, data) total_processed += batch_processed logger.info(f"Processed batch {page}: {batch_processed} {entity_type} records") # Check if we got less than batch_size (last page) if len(data) < batch_size: break page += 1 logger.info(f"Full sync completed for {entity_type}: {total_processed} total records") return total_processed except Exception as e: logger.error(f"Full sync failed for {entity_type}: {str(e)}") raise async def _store_entity_data(self, entity_type: str, data: List[Dict[str, Any]]) -> int: """ Store entity data in the database. Args: entity_type: Type of entity being stored data: List of entity records from AMO CRM Returns: Number of records processed """ try: processed_count = 0 for record in data: # Process main entity data await self._store_main_entity(entity_type, record) # Process custom fields if "custom_fields_values" in record: await self._store_custom_fields(entity_type, record) # Process relationships (embedded data) if "_embedded" in record: await self._store_relationships(entity_type, record) processed_count += 1 # Update last sync timestamp await self._update_last_sync_timestamp(entity_type) logger.info(f"Stored {processed_count} {entity_type} records in database") return processed_count except Exception as e: logger.error(f"Failed to store {entity_type} data: {str(e)}") raise async def _store_main_entity(self, entity_type: str, record: Dict[str, Any]) -> None: """ Store main entity record in the appropriate table. Args: entity_type: Type of entity record: Entity record data """ # TODO: Implement database insertion based on entity type # This would insert/update records in tables like amo_deals, amo_contacts, etc. entity_id = record.get("id") logger.debug(f"Storing {entity_type} record ID: {entity_id}") # Placeholder implementation pass async def _store_custom_fields(self, entity_type: str, record: Dict[str, Any]) -> None: """ Store custom fields for an entity. Args: entity_type: Type of entity record: Entity record containing custom fields """ entity_id = record.get("id") custom_fields = record.get("custom_fields_values", []) for field in custom_fields: # TODO: Implement custom field storage in amo_custom_fields table field_id = field.get("field_id") field_name = field.get("field_name", f"field_{field_id}") values = field.get("values", []) logger.debug(f"Storing custom field {field_name} for {entity_type} ID: {entity_id}") # Placeholder implementation pass async def _store_relationships(self, entity_type: str, record: Dict[str, Any]) -> None: """ Store entity relationships from embedded data. Args: entity_type: Type of entity record: Entity record containing embedded relationships """ entity_id = record.get("id") embedded = record.get("_embedded", {}) for relation_type, relations in embedded.items(): if not isinstance(relations, list): continue for relation in relations: # TODO: Implement relationship storage in junction tables relation_id = relation.get("id") is_main = relation.get("is_main", False) logger.debug(f"Storing {relation_type} relationship: {entity_type} {entity_id} -> {relation_id}") # Placeholder implementation pass async def _get_last_update_timestamp(self, entity_type: str) -> Optional[int]: """ Get the timestamp of the last successful sync for an entity type. Args: entity_type: Type of entity Returns: Unix timestamp of last update or None for full sync """ # TODO: Implement database query to get last sync timestamp # This could be stored in a sync_status table or derived from entity updated_at logger.debug(f"Getting last update timestamp for {entity_type}") # Placeholder - return None for full sync return None async def _update_last_sync_timestamp(self, entity_type: str) -> None: """ Update the last sync timestamp for an entity type. Args: entity_type: Type of entity """ # TODO: Implement database update for last sync timestamp current_time = datetime.now(timezone.utc) logger.debug(f"Updating last sync timestamp for {entity_type} to {current_time}") # Placeholder implementation pass def _is_valid_entity_type(self, entity_type: str) -> bool: """ Validate if the entity type is supported. Args: entity_type: Type of entity to validate Returns: True if valid, False otherwise """ valid_types = { "deals", "contacts", "companies", "users", "pipelines", "events" } return entity_type in valid_types