
    !+i                        d Z ddlZddlmZmZmZ ddlmZ ddlmZm	Z	m
Z
 ddlmZmZ ddlmZ ddlmZ dd	lmZmZmZmZ dd
lmZmZ ddlmZ ddlZ ej:                  e      ZddZ ddZ!d dZ"dde	e   de#defdZ$dede%de%de&de'defdZ(defdZ)dede%de%de&de'defdZ*dde	e   de#defdZ+dede%de%de&de'defdZ,d Z-y)!zn
Hyper Forecast - Learning Jobs
Daily reconciliation and weekly auto-calibration jobs for the learning system
    N)datetimedate	timedelta)Decimal)DictListTuple)and_func)Session)SessionLocal)ForecastLogCalibrationHistoryMultiplierConfigAllowedFactor)MlOrderMlOrderItem)	SystemLogc                 @   t         j                  d       t               }	 ddlm} ddlm}m}  ||      } |j                         }d}g }| rt         j                  d       |j                         }	|	 |d      z   }
t        d      D ]F  }|j                   |j                  |	|j                  j                                ||	      z          H t        d      D ]F  }|j                   |j                  |
|j                  j                                ||	      z          H t         j                  d
t        |       d       nt         j                  d       |j                          |d      z   }
t        d      D ]F  }|j                   |j                  |
|j                  j                                ||	      z          H t         j                  d|
        |D ]  }	 |j                  t               j#                  t         j$                  |k(        j'                         }|r)t         j)                  d|j+                  d       d       r|j,                  }|j                         }|j/                  ||      }|r7d|v r3|dz  }t         j                  d|j+                  d       d|d   d        t         j                  d| d       d||j                         j5                         d|j7                          S # t0        $ r&}t         j3                  d| d|        Y d}~Pd}~ww xY w# t0        $ r@}t         j3                  d|        dt9        |      dcY d}~|j7                          S d}~ww xY w# |j7                          w xY w)a8  
    Daily job - runs at 00:00 (midnight) OR manually
    Generate predictions for the next 24 hours (or entire days if manual).
    
    Args:
        manual_run: If True, generates for ALL hours of today + tomorrow (48 total)
                   If False (scheduled), generates only for tomorrow's 24 hours
    z6[FORECAST-JOB] Starting daily prediction generation...r   )HyperForecast)r   r   zK[FORECAST-JOB] MANUAL RUN: Generating for TODAY + TOMORROW (48 hours total)   days   hoursz[FORECAST-JOB] Target: z% hours (today 00-23 + tomorrow 00-23)zE[FORECAST-JOB] SCHEDULED RUN: Generating for TOMORROW only (24 hours)z$[FORECAST-JOB] Target: 24 hours for z[FORECAST-JOB] Skipping z%Y-%m-%d %Hhz - already exists
predictionz[FORECAST-JOB] z: R$ z.2fz[FORECAST-JOB] Failed for : Nu   [FORECAST-JOB] ✓ Complete: z predictions generatedok)statuspredictions_madetarget_datez)[FORECAST-JOB] Daily predictions failed: errorr    message)loggerinfor   app.services.forecast.enginer   r   r   nowr   rangeappendcombinemintimelenqueryr   filter	hora_alvofirstdebugstrftimehourpredict_hour_with_logging	Exceptionr#   	isoformatclosestr)
manual_rundbr   r   r   forecastr)   r!   target_hourstodaytomorrowr6   	target_dtexistinghour_num	date_partresulte_houres                      D/var/www/hypershopcomercio.com.br/hyper-ai/app/jobs/forecast_jobs.pyrun_daily_predictionsrJ      sT    KKHI	BM>0 $hllnKKefHHJEya00H b	 j##$4H$4$4UHLL<M<M<O$PS\cgSh$hij b	 m##$4H$4$4Xx||?P?P?R$SV_fjVk$klm KK1#l2C1DDijk KK_`xxzI1$55H b	 m##$4H$4$4Xx||?P?P?R$SV_fjVk$klm KK>xjIJ & 	I88K077))Y6%'  LL#;I<N<N~<^;__p!qr %>>%NN,	!;;HiPlf4$)$KK/)2D2D^2T1UUZ[abn[opsZt uv'	2 	34D3EE[\]  088://1
 	
#  9)BvhOP  6@DE!c!f55 	
6
 	
sh   G	L? +A-LL? A'L <L? 	L<L71L? 7L<<L? ?	N%N-N.N NN Nc                 d   t         j                  d|  d       t               }	 ddlm}  |t        d            }t        j                         }|j                  |      }| r	t        | t              r$t        j                  | d      j                         } t        j                  | t        j                  j                               }t        j                  | t        j                  j                               }|j!                  t"              j%                  t'        t"        j(                  |k\  t"        j(                  |k  t"        j(                  |k              j+                         }nz|t        d	
      z
  }	|j!                  t"              j%                  t'        t"        j(                  |	k\  t"        j(                  |t        d      z
  k              j+                         }|s+t         j                  d       dddd|j-                          S 	 i }
|D ]+  }|j(                  }||
vrg |
|<   |
|   j/                  |       - d}|
j1                         D ]c  \  }}t3        |      dkD  s|j5                  d d       |d   }|dd D ].  }|j7                  |       |dz  }||v s|j9                  |       0 e |dkD  r)t         j                  d| d       |j;                          t         j                  dt3        |       d       d}d}|D ]W  }	 |j(                  }|t        d      z   }|j                  |      }|j                  |      }|jA                  |jB                        j                  d      }|jA                  |jB                        j                  d      }|j!                  tE        jF                  tH        jJ                              j%                  t'        tH        jL                  |k\  tH        jL                  |k  tH        jN                  jQ                  g d                  jS                         }tU        |xs d      }t         jW                  d|j(                   d|        tU        |jX                  xs d      }tU        |jZ                  xs d      }|dkD  r||z
  |z  dz  }n	|dk(  rdnd}|dk(  xr |jZ                  du }| xr t]        ||z
        dkD  } |j^                  rd|j^                  v r
	 |j!                  t`        jb                  tE        jF                  t`        jd                        jg                  d      tE        jF                  t`        jh                  t`        jd                  z        jg                  d            jk                  tH              j%                  t'        tH        jL                  |k\  tH        jL                  |k  tH        jN                  jQ                  g d                  jm                  t`        jb                        j+                         }!|!D "ci c]8  }"|"jb                  to        |"jp                        tU        |"jr                        d : }#}"tu        |j^                        }$|$d   }%d!}&|%D ]q  }'|'jw                  d"      xs$ |'jw                  d#      xs |'jw                  d$      }(|(r|(|#v r|#|(   d   |'d%<   |#|(   d   |'d&<   d}&Yd%|'vs	|'d%   dk7  sfd|'d%<   d'|'d&<   d}&s |&r|$|_/        t}        t        t        |d*                  |_-        t}        t        t        |d*                  |_@        |dz  }| r|dz  }Z |j;                          t         j                  d.| d/| d0       	 d}+d},|D ]9  }|jZ                  |+t]        tU        |j                  xs d            z  }+|,dz  },; |,dkD  r|+|,z  nd}-t        d1d2d3| d4t        j                  d5|t        |-d*      d6d7      dd89      }.|j                  |.       |j;                          d|d;|j-                          S # t<        $ r#}t         j?                  d|        Y d}~d}~ww xY wc c}"w # t<        $ r0})t         jy                  d(|jz                   d)|)        Y d})~)d})~)ww xY w# t<        $ r=}*t         j?                  d+|jz                   d,|j(                   d-|*        Y d}*~*d}*~*ww xY w# t<        $ r"}*t         j?                  d:|*        Y d}*~*d}*~*ww xY w# t<        $ rP}/t         j?                  d<|/        |j                          d=t        |/      d>cY d}/~/|j-                          S d}/~/ww xY w# |j-                          w xY w)?a
  
    Hourly job - runs every hour (e.g., at :05)
    
    Re-reconciles ALL past hours of TODAY to catch late-arriving sales.
    Hours from previous days are left unchanged (finalized).
    Updates forecast_logs with valor_real and erro_percentual.

    Args:
        target_date: Optional. If string ('YYYY-MM-DD') or date object, specific date to reconcile.
                    If provided, bypasses the "only past hours of today" logic and processes ALL hours for that date
                    where we have data.
    z7[FORECAST-JOB] Starting hourly reconciliation (Target: )...r   )timezoner   )tzinfo%Y-%m-%d   r   r   z+[FORECAST-JOB] No pending logs to reconciler   )r    
reconciledupdatedc                 6    | j                   d u| j                  fS N)
valor_realidxs    rI   <lambda>z+run_hourly_reconciliation.<locals>.<lambda>   s    all$.F-M     T)keyreverseNz[FORECAST-JOB] Auto-removed z duplicate logsz5[FORECAST-JOB] Duplicate cleanup failed (non-fatal): z[FORECAST-JOB] Reconciling z  pending hours from last 30 days)paidshipped	deliveredz[DEBUG-RECONCILE] Processing z | Revenue: d   {Gz?_product_mixqtyrev)rd   re   Fmlb_id
product_idrW   realized_unitsrealized_revenueg        z2[FORECAST-JOB] Deep reconciliation failed for log r      z'[FORECAST-JOB] Failed to reconcile log z (z): z([FORECAST-JOB] Reconciliation complete: z new, z updatedhyper_aiINFOu   Conciliação: u    previsões processadasreconciliationhourly)actioncountavg_abs_errorperiodsuccess)modulelevelr%   detailsduration_msr    Failed to write system log: )r    rR   z&[FORECAST-JOB] Reconciliation failed: r#   r$   )Fr&   r'   r   r   rM   r   r)   replace
isinstancer;   strptimer   r,   r-   r.   maxr0   r   r1   r
   r2   allr:   r+   itemsr/   sortdeleteremovecommitr8   r#   
astimezoneutcr   sumr   total_amountdate_closedr    in_scalarfloatr4   valor_previstorV   absfatores_usadosr   
ml_item_idquantitylabel
unit_pricejoingroup_byintrd   re   dictgetwarningrW   r   rounderro_percentualr   jsondumpsaddrollback)0r"   r=   rM   tz_brr)   	now_local	day_startday_endlogs_to_processcutoff_datehour_countslhduplicates_removedgroupkeeperremovere_dupreconciled_countupdated_countlog
hour_starthour_endhour_start_localhour_end_localhour_start_utchour_end_utcactual_revenue	predictedold_realerro	was_emptywas_updated
sold_itemsitemsold_mapfactors_copymixrS   ppiddeep_erre_logtotal_abs_errorrp   rq   sys_logrH   s0                                                   rI   run_hourly_reconciliationr   t   s	    KKI+VZ[\	B\%,-llnKKuK-	 +s+&//ZHMMO !((hll6G6G6IJI&&{HLL4E4E4GHG !hh{3::))Y6))W4  ))C/ ce  	r 22K hh{3::))[8))C)!2D,DD ce  KKEF"!B\ 	
U	ZK$ )KKK'"QA%%a()
 "#'--/ 
<5u:>JJ#MW[J\"1XF#(9 <		'**a/*"o5+227;<
< "A%:;M:No^_		 	1#o2F1GGghi" ]	C\ ]]
%	(:: $.#5#5U#5#C !)!1!1!1!?!1!<!<X\\!J!R!RZ^!R!_-88FNNVZN[!#$((73G3G*H!I!P!P++~=++l:**+KL" &(  "'~':!;<S]]O<XfWghi!#"4"4"9:	 !415 "A%&7>ISPD )Q1CD %MDcnn.D	"+mUNX4M0NQU0U %%.C<N<N*N*r%'XX'22 HH[%9%9:@@G HH[%;%;k>R>R%RSYYZ_`& $w-  ' 3 3~ E ' 3 3l B ' 2 23S T) #(;#9#9:335 # q{#{hlDOOS]SXY]YaYaSb5c$c#{#{ (,C,>,>'?*>:"'!$ 3A"#%%/"WQUU<5H"WAEERVKC"sh6>smE6J"2 38@e8L"4 5*. $41#<BR@SWX@X:;A&6$7<?A&8$9.2G3 #1=C. ")U>1-E)F!G&-c%a..A&B# A% !Q&Ms]	~ 			>?O>PPVWdVeemno	B _U' NN.#s51D1D1I+J'KKOQJE
 9>	_u4q] "*+;*<<ST/.&+M1&='	%  !W VVG_YY[
 *
 	
g  	ZLLPQVPWXYY	Z| $|4 % r)[\_\b\b[ccefneo'pqqr  FsvvhbQTQ^Q^P__bchbijkD  	B\\8@AA	B  6=aSAB
!c!f55 	
6 	
s	  G e Aa 0;a ,Aa /,e G7c
D/b=b	 BbbAc
//e d 5Bd e 	b#b;e be 	b	c%c<c
cc

	d2de de 	d>d94e 9d>>e 	f
5f?f f ff f/c           	         t         j                  d|  d| d       t               }	 |j                  t              j                  t        t        j                  j                  d      t        j                  j                  d                  }|rt        |t              r$t        j                  |d      j                         }t        j                  |t        j                   j#                               }t        j                  |t        j$                  j#                               }|j                  t        t        j&                  |k\  t        j&                  |k              }| s"|j                  t        j(                  dk7        }|j+                         }t-        |      dk  rC| sAt         j                  dt-        |       d	       d
dt-        |      d|j/                          S t         j                  dt-        |       d       t1        ||       }g }|j3                         D ]g  \  }	}
|
j3                         D ]O  \  }}|d   }|d   }|dk  r| s| rdnd}t5        |      |kD  s-t7        ||	|||      }|s?|j9                  |       Q i t;        ||       }|j3                         D ]l  \  }	}
|
j3                         D ]T  \  }}|d   }|d   }|dk  r| s| rdnd}t5        |      |kD  s-t=        ||	|||      }|s?d|d<   |j9                  |       V n |r=	 |D ]6  }t?        |d      rd|_        t?        |d      s#|j@                  r0||_         8 	 |jG                          t         j                  dt-        |       d       dt-        |      |d|j/                          S # tB        $ r"}t         jE                  d|        Y d}~wd}~ww xY w# tB        $ rP}t         jI                  d|        |jK                          dt        |      dcY d}~|j/                          S d}~ww xY w# |j/                          w xY w) a8  
    Hourly micro-calibration job (renamed from weekly for historical reasons).
    
    Analyzes error patterns from the past 24 hours (or specific target date).
    Makes small adjustments (max 1% per cycle) for faster learning.
    Minimum 3 samples required per factor.
    
    Args:
        force_run: If True, bypasses sample size limits and forces calibration.
                   Also allows re-calibrating logs that were already marked calibrated.
        target_date: Optional. Specific date to calibrate (overrides 'uncalibrated only' logic if forced).
    z9[FORECAST-JOB] Starting hourly micro-calibration (Force: z, Date: rL   NrP   Y   z3[FORECAST-JOB] Not enough samples for calibration (z < 3)skippedinsufficient_samples)r    reasonsamplesz[FORECAST-JOB] Analyzing z predictions from last 24h...)	force_run	avg_errorrp   g?g      @Tis_product_factor
calibratedcalibration_impactz2[FORECAST-JOB] Could not mark logs as calibrated: z+[FORECAST-JOB] Micro-calibration complete: z adjustmentsr   )r    total_samplesadjustmentsz#[FORECAST-JOB] Calibration failed: r#   r$   )&r&   r'   r   r0   r   r1   r
   rV   isnotr   rz   r;   r   r{   r   r,   r-   r.   r|   r2   r   r}   r/   r:   _analyze_errors_by_factorr~   r   _apply_micro_calibrationr+   _analyze_product_factor_errors!_apply_product_factor_calibrationhasattrr   r8   r   r   r#   r   )r   r"   r=   r0   r   r   logsfactor_errorsadjustments_madefactor_typefactor_data
factor_keystatsr   sample_count	threshold
adjustmentproduct_factor_errorsr   col_errrH   s                        rI   run_weekly_calibrationr   d  s    KKKI;V^_j^kkopq	Bv%,,&&,,T2++11$7
 +s+&//ZHMMO ((hll6G6G6IJI&&{HLL4E4E4GHGLL))Y6))W4E LL!7!73!>?Eyy{t9q=KKMcRVi[X]^_'3IVYZ^V_`h 	
e 	/D	{:WXY 2$)L (5(;(;(= 	<$K%0%6%6%8 <!
E!+.	$W~  !#I $-C#	y>I-!9KY"J "(//
;'<	<4 !?ty Y(=(C(C(E 	<$K%0%6%6%8 <!
E!+.	$W~!#I#,C#	y>I-!BKY"J ":>
#67(//
;<	<& _ BCsL1),s$89#BXBX1A.B 			A#FVBWAXXdef  Y+
 	
'  _!ST[S\]^^_  6:1#>?
!c!f55 	
6 	
s   F3O! ,BO! 0O! A4O! 7O! 	O! &$N3 N3 	N3 !AO! 3	O<OO! OO! !	P:*5P5P: P= 5P::P= =Qr   r   returnc                    i i i i i i i i i i d
}t               }|j                  t              j                  t        j                  dk(        j                         }|j                          i }|D ]O  }|j                  |vrt               ||j                  <   ||j                     j                  |j                         Q |sh dh dh dh dh dh dd	}| D ]  }|j                  xs i }t        |j                  xs d
      }	t        |j                               D ]z  }
d|
 }||vrt!        ||         }|s|
|v r|||
   vr)t#        |      dkD  sd|v r<|||
   vrg d
d||
   |<   ||
   |   d   j%                  |	       ||
   |   dxx   dz  cc<   |  |D ]D  }
||
   D ]:  }||
   |   d   }|rt'        |      t#        |      z  nd
||
   |   d<   ||
   |   d= < F |S )z
    Group prediction errors by factor type to identify patterns.
    CRITICAL: Only uses categorical metadata keys (_meta_*), never numeric values.
    )
day_of_weekmomentumperiod_of_montheventseasonalhourly_patternmobile_hoursimpulse_hoursweek_of_monthpayment_dayr   >   sextatercaquartaquintasabadodomingosegunda>   updownnormaldefaultneutral>   fimmeioinicio>   veraoneutrooutonoinverno	primavera>   dia_15dia_20r   quinto_dia_util>   12345)r   r   r   r   r   r   r   _meta_rQ    errorsrp   r  rp   r   r   )r   r0   r   r1   	is_activer}   r:   r   setr   r   r   r   r   listkeysr;   r/   r+   r   )r   r   r   r=   allowed_factors_dbALLOWED_KEYSafr   fatoresr   r   meta_keycategorical_keyr\   r  s                  rI   r   r     s^    M 
B-0778O8OSV8VW[[]HHJ L  8>>-+.5L(R^^$((78 aF8ML6
  !F$$*S((-A. 2 2 45 	FK}-Hw&!'("34O
 !<",{*CC ?#b(C?,B mK&@@IKVW>Xk*?; +&7AHHN+&7@AE@;	F	!FH % : - 	:C";/4X>FX^3v;V;TdeM+&s+K8k*3/9	:: r[   r=   r   r   r   r   c                    | j                  t              j                  t        t        j                  |k(  t        j
                  |k(              j                         }|rt        |j                        nd}d}|dkD  r	|d|z
  z  }n|dk  r	|d|z   z  }nyt        dt        d|            }|r<t        t        t        |d	                  |_        d
|_        t        d|      |_        nGt        ||t        t        t        |d	                  d
t        d|            }| j!                  |       t#        t%        j&                         ||t        t        t        |d	                  t        t        t        |d	                  t        t        t        |d                  |t        t        t        ||z  dz
  dz  d                  d|dd| 	      }	| j!                  |	       	 t)        ddd| d| t+        j,                  d| d| t        t        |d	            t        t        |d	            t        t        ||z  dz
  dz  d            t        t        |d            |d      d      }
| j!                  |
       t0        j5                  d| d| d|dd|dd|dd        |||t        |d	      t        |d      |d!S # t.        $ r"}t0        j3                  d|        Y d}~kd}~ww xY w)"z8
    Apply a calibration adjustment to a multiplier
          ?g{Gz?   r   N      ?       @r   autora   tipochavevalor	calibrado	confiancarj   zAuto-calibration: avg_error=.1f
% samples=	data_calibracao
tipo_fatorfator_chavevalor_anterior
valor_novo
erro_medioamostrasajuste_percentualnotasrk   rl   u   Calibração: .calibration)ro   factor	old_value	new_valuechange_percentr   r   rs   )rt   ru   r%   rv   r    rx   z[FORECAST-JOB] Calibrated r   .3f -> z (error=z%)r   r   r:  r;  r   r   )r0   r   r1   r
   r&  r'  r3   r   r(  r|   r-   r   r;   r   r)  r*  r   r   r   utcnowr   r   r   r8   r&   r#   r'   )r=   r   r   r   r   configr:  adjustment_factorr;  historyr   r   s               rI   _apply_calibrationrD  E  s    XX&'..!![0""j0	

 eg  (.fll#3I
 1}%6!67		R%6!67	 CS),-I s5A#678!sL1!#eIq123#|,
 	v ! )s5A#6783uY2343uY234!#eY-BQ-F#,Mq&Q"RS,YsO:l^T
G FF7O=%k]!J<@ZZ()]!J<8#E)Q$78#E)Q$78#(	I0E0IS/PRS)T#U#E)Q$78(!  
 	w KK,[M:,bSVW[\efi[jjrs|  ~A  sB  BD  E  F # 9a(9a(   =3E7;<<=s   *BK 	K6K11K6c                  ^   t               } 	 | j                  t              j                         }| j                  t              j	                  t        j
                  j                  d            j                         }t        j                         t        d      z
  }| j                  t              j	                  t        t        j                  |k\  t        j                  j                  d                  j                         }d}|rt        d |D              t        |      z  }| j                  t               j#                  t         j$                  j'                               j)                  d      j                         }| j                  t*              j                         }||||z
  t-        |d      |D cg c]f  }|j$                  j/                         |j0                   d|j2                   |j4                   d	|j6                   t9        |j:                        d
h c}|D cg c]E  }|j<                  |j>                  t9        |j@                        |jB                  |jD                  dG c}d| jG                          S c c}w c c}w # | jG                          w xY w)z1Get current calibration status and recent historyN   r   r   c              3   N   K   | ]  }t        |j                  xs d         ywr   N)r   r   .0r   s     rI   	<genexpr>z)get_calibration_status.<locals>.<genexpr>  s      OaE!"3"3"8q9O   #%
   rj   r7  r>  )r   r9  changer#   )typer\   valuesource
confidence)total_predictions_loggedpredictions_reconciledpending_reconciliationavg_error_7drecent_calibrationscurrent_multipliers)$r   r0   r   rp   r1   rV   r   r   r)   r   r
   timestamp_previsaor   r}   r   r/   r   order_byr.  desclimitr   r   r9   r/  r0  r1  r2  r   r3  r&  r'  r(  r)  r*  r:   )	r=   
total_logsreconciled_logsweek_agorecent_logsr   rW  configscs	            rI   get_calibration_statusrc    s/   	B5XXk*002
((;/66""((.

%' 	
 <<>I1$55hh{+22..(:++11$7

 #% 	 	O;OORUVaRbbI !hh'9:CC..335

%)CCE 	
 ((+,002 )3&5&0?&B!)Q/ /$ 	 --779!"a?!"!1!1 2$q||nE"1<<0	$ #$  FF77"177^kk"#++$
2 	
)$$ 	
s,   F4J  A+J+J 1A
J;J 
J J,c                    | j                  t              j                  t        t        j                  |k(  t        j
                  |k(              j                         }|r,t        |dd      dk(  rt        j                  d| d|        y|rt        |j                        nd}d}|d	kD  r	|d
|z
  z  }n|dk  r	|d
|z   z  }nyt        dt        d|            }|r<t        t        t!        |d                  |_        d|_        t        d|      |_        nGt        ||t        t        t!        |d                  dt        d|            }| j'                  |       t)        t+        j,                         ||t        t        t!        |d                  t        t        t!        |d                  t        t        t!        |d                  |t        t        t!        ||z  d
z
  dz  d                  d|dd| 	      }	| j'                  |	       t        j/                  d| d| d|dd|dd	       |||t!        |d      t!        |d
      |dS )z
    Apply a MICRO calibration adjustment (max 1% per cycle) for faster learning.
    Respects the 'locked' flag - skips calibration if factor is locked.
    lockedNr   z'[FORECAST-JOB] Skipping locked factor: r7  Nr  rb   r   r   r!  r"  r#  r   r$  ra   r%  rj   z&Micro-calibration (hourly): avg_error=r+  r,  r-  z [FORECAST-JOB] Micro-calibrated r   r=  r>  u    (±1%)r?  )r0   r   r1   r
   r&  r'  r3   getattrr&   r4   r   r(  r|   r-   r   r;   r   r)  r*  r   r   r   r@  r'   )
r=   r   r   r   r   rA  r:  rB  r;  rC  s
             rI   r   r     sN    XX&'..!![0""j0	

 eg  '&(C0C7>{m1ZLYZ'-fll#3I 1}%6!67		R%6!67	 CS),-I s5A#678!sL1!#eIq123#|,
 	v ! )s5A#6783uY2343uY234!#eY-BQ-F#,Mq&Q"RS6yoZP\~^
G FF7O
KK2;-qByY\o]abkloappwxy # 9a(9a( r[   c                 H   g d}i }|D ]  }i ||<   	 | D ]@  }|j                   xs i }t        |j                  xs d      }|j                  dg       }|D ]  }	|	j                  di       }
|D ]  }||
v st        |
|         }|dk(  r|dk  rd}n|dk  rd	}n{|d
k  rd}nsd}np|dv r
|dkD  rdnd}nb|dk(  r|dk  rd}nU|dkD  rd}nMd}nJ|dk(  r|dk\  rd}n=|d
k\  rd}n5d}n2|dk(  r|dk\  rd}n%|dk\  rd}n|dk\  rd}nd }n|d
k  rd}n
|dkD  rd!}nd}|||   vrg dd"||   |<   ||   |   d#   j	                  |       ||   |   d$xx   d%z  cc<     C |D ]D  }||   D ]:  }||   |   d#   }|rt        |      t        |      z  nd||   |   d&<   ||   |   d#= < F |S )'z
    Analyze prediction errors grouped by PRODUCT-specific factors.
    These are factors that vary by product (stock_pressure, catalog_boost, etc).
    
    Uses the product_multipliers stored in each product's _product_mix entry.
    )stock_pressurecatalog_boostlisting_healthfree_shippingshipping_advantagelisting_type
gold_medalpromo_activevelocity_scoretop_sellersvisits_trendconversion_rateprice_competitivenesssearch_positionr   rc   product_multipliersri  333333?
zero_stockg333333?criticalg?lowr   )rj  ro  rp  rl  rr  r  activeinactivers  g?r   g333333?r   r   rq  g?ABCrv  top5top10top20below20highr  r  rp   r   r   )r   r   r   r   r+   r   r/   )r   r   PRODUCT_FACTOR_TYPESr   ftr   r  r   product_mixprodrw  r   rP  
bucket_keyr\   r  s                   rI   r   r   3  s   $ M" b  AI$$*S((-A. kk."5 :	ID"&((+@""E  4 6I"55!"5k"BCE #&66 C<)5J"c\)3J"c\).J)1J$(ww16X*
$6 3;)/J"S[)-J)2J$(88 C<),J"c\),J),J$(99 C<)/J"c\)0J"c\)0J)2J !3;).J"S[)/J)2J "{)CCLNYZA[k2:>!+.z:8DKKDQ!+.z:7CqHCm6I	:	IAIH % : - 	:C";/4X>FX^3v;V;TdeM+&s+K8k*3/9	:: r[   c                    ddl m}m} d| }| j                  |      j	                  t        |j                  |k(  |j                  |k(              j                         }|r,t        |dd      dk(  rt        j                  d| d|        y	|rt        |j                        nd
}	d}
|dkD  r	|	d|
z
  z  }n|dk  r	|	d|
z   z  }ny	t        dt        d|            }|r<t!        t#        t%        |d                  |_        d|_        t        d|      |_        nD |||t!        t#        t%        |d                  dt        d|            }| j+                  |        |t-        j.                         ||t!        t#        t%        |	d                  t!        t#        t%        |d                  t!        t#        t%        |d                  |t!        t#        t%        ||	z  dz
  dz  d                  d| d| d|dd	      }| j+                  |       t        j1                  d| d| d|	dd|d       |||	t%        |d      t%        |d      |dS ) z
    Apply a calibration adjustment to a PRODUCT-specific multiplier.
    Uses the same MultiplierConfig table but with 'product_' prefix on type.
    r   )r   r   product_re  rf  r   z/[FORECAST-JOB] Skipping locked product factor: r7  Nr  rb   r   r   r!  rx  r#  r   r$  ra   r%  rj   zProduct-factor calibration: z avg_error=r+  %r-  z)[FORECAST-JOB] Calibrated PRODUCT factor r   r=  r>  r?  )app.models.forecast_learningr   r   r0   r1   r
   r&  r'  r3   rg  r&   r4   r   r(  r|   r-   r   r;   r   r)  r*  r   r   r@  r'   )r=   r   r   r   r   r   r   db_factor_typerA  r:  rB  r;  rC  s                rI   r   r     sq    R  }-N XX&'..!!^3""j0	

 eg  '&(C0C7F~FVVWXbWcde'-fll#3I 1}%6!67		R%6!67	 CS),-I s5A#678!sL1!#eIq123#|,
 	v ! )!s5A#6783uY2343uY234!#eY-BQ-F#,Mq&Q"RS,[M:,kR[\_Q``ab
G FF7O
KK;K=*UWXabeWffjktuxjyz{ # 9a(9a( r[   c            	      2   ddl m}  t        j                  d       t	               }	 t        j                         }t        j                  |t        j                  j                               }t        j                  |t        j                  j                               }|j                  t              j                  t        t        j                   |k\  t        j                   |k  t        j"                  j%                  d                  j'                         }|s*t        j                  d       ddd|j)                          S t+        d	 |D              }t+        d
 |D              }|D cg c](  }|j,                  t/        |j,                  xs d      * }	}|	rt+        |	      t1        |	      z  nd}
|	rt+        d |	D              t1        |	      z  nd}t        dd|z
        }i }|D ]  }|j2                  st/        |j,                  xs d      }|j2                  j5                         D ]X  \  }}||vri ||<   t7        |      }|||   vrg dd||   |<   ||   |   d   j9                  |       ||   |   dxx   dz  cc<   Z  |D ]B  }||   D ]8  }||   |   d   }|r!t;        t+        |      t1        |      z  d      nd||   |<   : D g }|j5                         D ]>  \  }}|j5                         D ]&  \  }}|j9                  | d| t=        |      f       ( @ |rt        |d       d   nd}|rt        |d       d   nd}|j                  t>              j                  t>        j@                  |k\        j'                         }|D cg c]M  }|jB                   d|jD                   |jF                   d|jH                   t/        |jJ                        dO }}|j                  |       j                  | jL                  |k(        jO                         }|s | |      }|jQ                  |       t1        |      |_)        tU        t7        t;        |
d                  |_%        tU        t7        t;        |d                  |_+        tU        t7        t;        |d                  |_,        tU        t7        t;        |d                  |_-        tU        t7        t;        |d                  |_.        ||_/        t1        |      |_0        ||_1        ||_2        ||_3        |ji                          t        j                  d| d|dd       d|jk                         t;        |d      t1        |      t1        |      d|j)                          S c c}w c c}w # tl        $ rP}t        jo                  d|        |jq                          d t7        |      d!cY d}~|j)                          S d}~ww xY w# |j)                          w xY w)"zv
    Daily job - runs at 23:55
    
    Creates a snapshot of the day's learning metrics for historical analysis.
    r   )LearningSnapshotz2[FORECAST-JOB] Creating daily learning snapshot...Nz,[FORECAST-JOB] No logs to snapshot for todayr   no_data)r    r   c              3   N   K   | ]  }t        |j                  xs d         ywrH  )r   r   rI  s     rI   rK  z%run_daily_snapshot.<locals>.<genexpr>  s      HaU1#3#3#8q9HrL  c              3   N   K   | ]  }t        |j                  xs d         ywrH  )r   rV   rI  s     rI   rK  z%run_daily_snapshot.<locals>.<genexpr>  s     @aq||0q1@rL  c              3   2   K   | ]  }t        |        y wrU   )r   )rJ  rH   s     rI   rK  z%run_daily_snapshot.<locals>.<genexpr>  s     4SV4s   ra   r  r  rp   r   rj   r7  c                     | d   S Nr    rX   s    rI   rZ   z$run_daily_snapshot.<locals>.<lambda>4  s
    AaD r[   )r\   c                     | d   S r  r  rX   s    rI   rZ   z$run_daily_snapshot.<locals>.<lambda>5  s
    !A$ r[   r>  )r9  rN  r#   )dataz'[FORECAST-JOB] Daily snapshot created: z - accuracy=r+  r  r   )r    r   accuracypredictionscalibrationsz&[FORECAST-JOB] Daily snapshot failed: r#   r$   )9r  r  r&   r'   r   r   r@   r   r,   r-   r.   r|   r0   r   r1   r
   r2   rV   r   r}   r:   r   r   r   r/   r   r~   r;   r+   r   r   r   r.  r/  r0  r1  r2  r3  r  r3   r   total_previsoesr   erro_absoluto_medioacuraciareceita_prevista_totalreceita_real_totalfatores_performanceajustes_realizadosdetalhes_ajustesmelhor_fator
pior_fatorr   r9   r8   r#   r   )r  r=   r@   today_start	today_endr   total_previsto
total_realr   r  r3  erro_abs_medior  r  r   r   r   factor_valuer\   r  errsall_factor_errorsr  errr  r  calibracoesrb  r  snapshotrH   s                                  rI   run_daily_snapshotr    s    >
KKDE	Bi

&&uhll.?.?.AB$$UHLL,=,=,?@	 xx$++%%4%%2&&,,T2
 #% 	 KKFG'9=r 	
m H4HH@4@@
9=_AARARA^%)).Q/__28S[3v;.a
FL4V44s6{BRSq#./ ! 
	HC!!S005A6141C1C1I1I1K H-K"*==;=+K8l+C"5k"BBKMXY@Z+K8='4S9(CJJ4P'4S9'BaGBH
	H & 	^B*2. ^*2.s3H=RVuSYT5JA/N\]#B',^	^ +113 	DHB JJL DS!((RD#-S)BCD	D IZs,.A!D_cFWS*?B]a
 hh1299..+=

#% 	 #

  \\N!AMM?;--.d1<<.Aq||,
 
 88,-445E5J5Je5STZZ\'U3HFF8#&t9 %c%
A*>&?@'.s53K/L'M$#Ch(:$;<*1#eNA6N2O*P'&-c%
A2F.G&H#':$&)+&6#$4! ,(
		=eWLQYZ]P^^_`a OO%h*t9,
 	
i `P
L  6=aSAB
!c!f55 	
6 	
sd   DV( (V( /VVA!V( >FV( AV#+F"V( 
V( (	X15W<&X'X <XX X)FrU   )FN).__doc__loggingr   r   r   decimalr   typingr   r   r	   
sqlalchemyr
   r   sqlalchemy.ormr   app.core.databaser   r  r   r   r   r   app.models.ml_orderr   r   app.models.system_logr   r   	getLogger__name__r&   rJ   r   r   boolr   r;   r   r   rD  rc  r   r   r   r  r  r[   rI   <module>r     s    . .  $ $ ! " * i i 4 + 			8	$Z~m`GTTD$5 T$ TSW Tn^^^ ^ 	^
 ^ 
^D9 9xMMM M 	M
 M 
M`hk): ht hX\ hVRRR R 	R
 R 
Rjur[   