133 lines
3.8 KiB
Python
133 lines
3.8 KiB
Python
"""
|
|
FastStream broker setup for AMO CRM service background processing.
|
|
|
|
This module sets up the Redis-based message broker and defines worker
|
|
functions for processing export jobs, data synchronization, and scheduled tasks.
|
|
"""
|
|
|
|
import logging
|
|
from typing import Dict, Any
|
|
from faststream import FastStream
|
|
from faststream.redis import RedisBroker
|
|
from utils.config import settings
|
|
|
|
# Configure logging
|
|
logging.basicConfig(level=settings.LOG_LEVEL)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Initialize Redis broker
|
|
broker = RedisBroker(settings.REDIS_URL)
|
|
app = FastStream(broker)
|
|
|
|
|
|
@broker.subscriber("export-jobs")
|
|
async def process_export_job(job_data: Dict[str, Any]) -> None:
|
|
"""
|
|
Process Google Sheets export jobs.
|
|
|
|
Args:
|
|
job_data: Dictionary containing job_id, configuration_id, and metadata
|
|
"""
|
|
logger.info(f"Processing export job: {job_data.get('job_id')}")
|
|
|
|
try:
|
|
from servers.export_server import ExportServer
|
|
|
|
export_server = ExportServer()
|
|
await export_server.process_export_job(job_data)
|
|
|
|
logger.info(f"Export job {job_data.get('job_id')} completed successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Export job {job_data.get('job_id')} failed: {str(e)}")
|
|
raise
|
|
|
|
|
|
@broker.subscriber("sync-jobs")
|
|
async def process_sync_job(sync_data: Dict[str, Any]) -> None:
|
|
"""
|
|
Process AMO CRM data synchronization jobs.
|
|
|
|
Args:
|
|
sync_data: Dictionary containing job_id, entity_type, and parameters
|
|
"""
|
|
logger.info(f"Processing sync job: {sync_data.get('job_id')} for {sync_data.get('entity_type')}")
|
|
|
|
try:
|
|
from servers.sync_server import SyncServer
|
|
|
|
sync_server = SyncServer()
|
|
await sync_server.process_sync_job(sync_data)
|
|
|
|
logger.info(f"Sync job {sync_data.get('job_id')} completed successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Sync job {sync_data.get('job_id')} failed: {str(e)}")
|
|
raise
|
|
|
|
|
|
@broker.subscriber("refresh-jobs")
|
|
async def process_refresh_job(entity_type: str) -> None:
|
|
"""
|
|
Process scheduled entity data refresh jobs.
|
|
|
|
Args:
|
|
entity_type: Type of entity to refresh (deals, contacts, companies, etc.)
|
|
"""
|
|
logger.info(f"Processing refresh job for entity type: {entity_type}")
|
|
|
|
try:
|
|
from servers.sync_server import SyncServer
|
|
|
|
sync_server = SyncServer()
|
|
await sync_server.refresh_entity_data(entity_type)
|
|
|
|
logger.info(f"Refresh job for {entity_type} completed successfully")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Refresh job for {entity_type} failed: {str(e)}")
|
|
raise
|
|
|
|
|
|
@broker.subscriber("failed-jobs")
|
|
async def handle_failed_jobs(job_data: Dict[str, Any]) -> None:
|
|
"""
|
|
Handle permanently failed jobs (dead letter queue).
|
|
|
|
Args:
|
|
job_data: Failed job data for logging and potential manual intervention
|
|
"""
|
|
logger.error(f"Job permanently failed: {job_data}")
|
|
|
|
# TODO: Implement notification system (email, Slack, etc.)
|
|
# TODO: Store failed jobs for manual review
|
|
|
|
# For now, just log the failure
|
|
job_id = job_data.get("job_id", "unknown")
|
|
job_type = job_data.get("job_type", "unknown")
|
|
error_message = job_data.get("error_message", "No error message provided")
|
|
|
|
logger.error(
|
|
f"DEAD LETTER QUEUE - Job ID: {job_id}, "
|
|
f"Type: {job_type}, Error: {error_message}"
|
|
)
|
|
|
|
|
|
@broker.after_startup
|
|
async def startup_handler() -> None:
|
|
"""Handle broker startup tasks."""
|
|
logger.info("FastStream broker started successfully")
|
|
logger.info(f"Connected to Redis: {settings.REDIS_URL}")
|
|
|
|
|
|
@broker.before_shutdown
|
|
async def shutdown_handler() -> None:
|
|
"""Handle broker shutdown tasks."""
|
|
logger.info("FastStream broker shutting down")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# This allows running the worker directly with: python workers/broker.py
|
|
import asyncio
|
|
asyncio.run(app.run())
|