290 lines
9.5 KiB
Python
290 lines
9.5 KiB
Python
"""
|
|
Sync server for processing AMO CRM data synchronization jobs.
|
|
|
|
This module handles synchronization of data from AMO CRM API to the local
|
|
SQLite database, including both full syncs and incremental updates.
|
|
"""
|
|
|
|
import logging
|
|
from typing import Dict, Any, List, Optional
|
|
from datetime import datetime, timezone
|
|
from adapters.amocrm_client import AmoCRMClient
|
|
from adapters.sqlite.database import get_db
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class SyncServer:
|
|
"""Server for processing AMO CRM data synchronization operations."""
|
|
|
|
def __init__(self):
|
|
"""Initialize SyncServer with AMO CRM client and database."""
|
|
self.amocrm = AmoCRMClient()
|
|
self.db = get_db
|
|
|
|
async def process_sync_job(self, sync_data: Dict[str, Any]) -> None:
|
|
"""
|
|
Process AMO CRM data synchronization job.
|
|
|
|
Args:
|
|
sync_data: Dictionary containing job_id, entity_type, and parameters
|
|
"""
|
|
job_id = sync_data["job_id"]
|
|
entity_type = sync_data["entity_type"]
|
|
parameters = sync_data.get("parameters", {})
|
|
|
|
try:
|
|
logger.info(f"Starting sync job {job_id} for {entity_type}")
|
|
|
|
# Validate entity type
|
|
if not self._is_valid_entity_type(entity_type):
|
|
raise ValueError(f"Invalid entity type: {entity_type}")
|
|
|
|
# Fetch data from AMO CRM
|
|
data = await self.amocrm.fetch_entity_data(entity_type, **parameters)
|
|
|
|
if not data:
|
|
logger.warning(f"No data received from AMO CRM for {entity_type}")
|
|
return
|
|
|
|
# Process and store data
|
|
processed_count = await self._store_entity_data(entity_type, data)
|
|
|
|
logger.info(f"Sync job {job_id} completed: {processed_count} records processed")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Sync job {job_id} failed: {str(e)}")
|
|
raise
|
|
|
|
async def refresh_entity_data(self, entity_type: str) -> None:
|
|
"""
|
|
Refresh all data for an entity type with incremental updates.
|
|
|
|
Args:
|
|
entity_type: Type of entity to refresh (deals, contacts, companies, etc.)
|
|
"""
|
|
try:
|
|
logger.info(f"Starting refresh for {entity_type} data")
|
|
|
|
# Validate entity type
|
|
if not self._is_valid_entity_type(entity_type):
|
|
raise ValueError(f"Invalid entity type: {entity_type}")
|
|
|
|
# Get last update timestamp for incremental sync
|
|
last_update = await self._get_last_update_timestamp(entity_type)
|
|
|
|
# Fetch updated data from AMO CRM
|
|
data = await self.amocrm.fetch_entity_data(
|
|
entity_type,
|
|
updated_at=last_update,
|
|
limit=250 # Process in batches
|
|
)
|
|
|
|
if data:
|
|
processed_count = await self._store_entity_data(entity_type, data)
|
|
logger.info(f"Refresh completed for {entity_type}: {processed_count} records updated")
|
|
else:
|
|
logger.info(f"No updates found for {entity_type}")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Refresh failed for {entity_type}: {str(e)}")
|
|
raise
|
|
|
|
async def full_sync_entity(self, entity_type: str, batch_size: int = 250) -> int:
|
|
"""
|
|
Perform a full synchronization of an entity type.
|
|
|
|
Args:
|
|
entity_type: Type of entity to sync
|
|
batch_size: Number of records to fetch per batch
|
|
|
|
Returns:
|
|
Total number of records processed
|
|
"""
|
|
try:
|
|
logger.info(f"Starting full sync for {entity_type}")
|
|
|
|
total_processed = 0
|
|
page = 1
|
|
|
|
while True:
|
|
# Fetch batch of data
|
|
data = await self.amocrm.fetch_entity_data(
|
|
entity_type,
|
|
limit=batch_size,
|
|
page=page
|
|
)
|
|
|
|
if not data:
|
|
break
|
|
|
|
# Process batch
|
|
batch_processed = await self._store_entity_data(entity_type, data)
|
|
total_processed += batch_processed
|
|
|
|
logger.info(f"Processed batch {page}: {batch_processed} {entity_type} records")
|
|
|
|
# Check if we got less than batch_size (last page)
|
|
if len(data) < batch_size:
|
|
break
|
|
|
|
page += 1
|
|
|
|
logger.info(f"Full sync completed for {entity_type}: {total_processed} total records")
|
|
return total_processed
|
|
|
|
except Exception as e:
|
|
logger.error(f"Full sync failed for {entity_type}: {str(e)}")
|
|
raise
|
|
|
|
async def _store_entity_data(self, entity_type: str, data: List[Dict[str, Any]]) -> int:
|
|
"""
|
|
Store entity data in the database.
|
|
|
|
Args:
|
|
entity_type: Type of entity being stored
|
|
data: List of entity records from AMO CRM
|
|
|
|
Returns:
|
|
Number of records processed
|
|
"""
|
|
try:
|
|
processed_count = 0
|
|
|
|
for record in data:
|
|
# Process main entity data
|
|
await self._store_main_entity(entity_type, record)
|
|
|
|
# Process custom fields
|
|
if "custom_fields_values" in record:
|
|
await self._store_custom_fields(entity_type, record)
|
|
|
|
# Process relationships (embedded data)
|
|
if "_embedded" in record:
|
|
await self._store_relationships(entity_type, record)
|
|
|
|
processed_count += 1
|
|
|
|
# Update last sync timestamp
|
|
await self._update_last_sync_timestamp(entity_type)
|
|
|
|
logger.info(f"Stored {processed_count} {entity_type} records in database")
|
|
return processed_count
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to store {entity_type} data: {str(e)}")
|
|
raise
|
|
|
|
async def _store_main_entity(self, entity_type: str, record: Dict[str, Any]) -> None:
|
|
"""
|
|
Store main entity record in the appropriate table.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
record: Entity record data
|
|
"""
|
|
# TODO: Implement database insertion based on entity type
|
|
# This would insert/update records in tables like amo_deals, amo_contacts, etc.
|
|
|
|
entity_id = record.get("id")
|
|
logger.debug(f"Storing {entity_type} record ID: {entity_id}")
|
|
|
|
# Placeholder implementation
|
|
pass
|
|
|
|
async def _store_custom_fields(self, entity_type: str, record: Dict[str, Any]) -> None:
|
|
"""
|
|
Store custom fields for an entity.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
record: Entity record containing custom fields
|
|
"""
|
|
entity_id = record.get("id")
|
|
custom_fields = record.get("custom_fields_values", [])
|
|
|
|
for field in custom_fields:
|
|
# TODO: Implement custom field storage in amo_custom_fields table
|
|
field_id = field.get("field_id")
|
|
field_name = field.get("field_name", f"field_{field_id}")
|
|
values = field.get("values", [])
|
|
|
|
logger.debug(f"Storing custom field {field_name} for {entity_type} ID: {entity_id}")
|
|
|
|
# Placeholder implementation
|
|
pass
|
|
|
|
async def _store_relationships(self, entity_type: str, record: Dict[str, Any]) -> None:
|
|
"""
|
|
Store entity relationships from embedded data.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
record: Entity record containing embedded relationships
|
|
"""
|
|
entity_id = record.get("id")
|
|
embedded = record.get("_embedded", {})
|
|
|
|
for relation_type, relations in embedded.items():
|
|
if not isinstance(relations, list):
|
|
continue
|
|
|
|
for relation in relations:
|
|
# TODO: Implement relationship storage in junction tables
|
|
relation_id = relation.get("id")
|
|
is_main = relation.get("is_main", False)
|
|
|
|
logger.debug(f"Storing {relation_type} relationship: {entity_type} {entity_id} -> {relation_id}")
|
|
|
|
# Placeholder implementation
|
|
pass
|
|
|
|
async def _get_last_update_timestamp(self, entity_type: str) -> Optional[int]:
|
|
"""
|
|
Get the timestamp of the last successful sync for an entity type.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
|
|
Returns:
|
|
Unix timestamp of last update or None for full sync
|
|
"""
|
|
# TODO: Implement database query to get last sync timestamp
|
|
# This could be stored in a sync_status table or derived from entity updated_at
|
|
|
|
logger.debug(f"Getting last update timestamp for {entity_type}")
|
|
|
|
# Placeholder - return None for full sync
|
|
return None
|
|
|
|
async def _update_last_sync_timestamp(self, entity_type: str) -> None:
|
|
"""
|
|
Update the last sync timestamp for an entity type.
|
|
|
|
Args:
|
|
entity_type: Type of entity
|
|
"""
|
|
# TODO: Implement database update for last sync timestamp
|
|
|
|
current_time = datetime.now(timezone.utc)
|
|
logger.debug(f"Updating last sync timestamp for {entity_type} to {current_time}")
|
|
|
|
# Placeholder implementation
|
|
pass
|
|
|
|
def _is_valid_entity_type(self, entity_type: str) -> bool:
|
|
"""
|
|
Validate if the entity type is supported.
|
|
|
|
Args:
|
|
entity_type: Type of entity to validate
|
|
|
|
Returns:
|
|
True if valid, False otherwise
|
|
"""
|
|
valid_types = {
|
|
"deals", "contacts", "companies",
|
|
"users", "pipelines", "events"
|
|
}
|
|
return entity_type in valid_types
|