o
    pkj7a                  $   @   s  U d Z ddlmZ ddlmZ ddlZddd	Zejejd
ddgej	ej
ddddZejdejdejdddddejdejdejdddddejdejdejddejdddddejdejdejddejd d!d"d#dddejd$ejdejddejd d!ejjd%d&dddejd'ejdejddejdddddejd(ejdejddejd d!d)d*ejd+dd,dddejd-ejdejd.d/dddejd0ejdejd.d/ejd d!d)d*ejd1d/d2d3id4dddejd5ejd6ejd7dejd8d9d:ejd;dd<ed=dddejd>ejd6ejd7dejd:ejd;dd?d@edAdddejdBejd6ejd7dejd:ejd;ddCdDd?d@edEdddejdFejd6ejd7dejd8d9d:ejd;dd?d@edGdddejdHejdIejd7dejd8dJdKdLd@dDedMdddejdNejdejddejejdOdPdQdddejdRejdejddejd d!dSd*d)dLdLdTdTdUejejejdVdWdXdYdZdddejd[ejdejddejd d!dSd*d)ejejd)ejejejdVdWdXdYd\gd]d^dddgZeej ed_< ejee d`edaZdbZ ej!j"dcdddedfdg Z#ej!j"dcdddedhdi Z$ej!j"dcdddedjdk Z%ej!j"dcdddedldm Z&ej!j"dcdddeej!j'dndo Z(dS )pzTests for tunings.tune().   )types   )pytest_helper    Nv1beta1z?https://us-central1-autopush-aiplatform.sandbox.googleapis.com/)api_versionbase_urlbleuztest prompt template)nameprompt_templategs://sararob_test/output_uri_prefixgcs_destination)metricsoutput_configtest_dataset_gcs_urigemini-2.5-flashzVgs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_train_data.jsonlgcs_uri)
base_modeltraining_datasetzDparameter is only supported in Gemini Enterprise Agent Platform mode)r
   
parametersexception_if_mldevtest_tune_pretuned_modelzFprojects/801452371447/locations/us-central1/models/9030969596621881344Vgs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_train_data.jsonl+test_tune_pretuned_model_with_checkpoint_id3)pre_tuned_model_checkpoint_idr   r   configtest_tune_simple_dpozModel display name   PREFERENCE_TUNING)tuned_model_display_nameepoch_countmethodtest_tune_dpo_with_beta      ?)r%   r&   r'   beta*test_non_pretuned_model_with_checkpoint_id#test_dataset_gcs_uri_all_parameters      ?ADAPTER_SIZE_ONEz[gs://cloud-samples-data/ai-platform/generative_ai/gemini-1_5/text/sft_validation_data.jsonl)r%   r&   learning_rate_multiplieradapter_sizevalidation_dataset$test_dataset_vertex_dataset_resourcezHprojects/801452371447/locations/us-central1/datasets/5779918772206829568)vertex_dataset_resource,test_dataset_dataset_resource_all_parameterszHprojects/801452371447/locations/us-central1/datasets/1168232753779441664testlabelkeytestlabelvalue)r%   r&   r/   r0   r1   labelstest_tune_distillationz#meta/llama3_1@llama-3.1-8b-instructzEgs://nathreya-oss-tuning-sdk-test/distillation-openai-opposites.jsonlDISTILLATIONzdeepseek-ai/deepseek-v3.1-maas   zIgs://nathreya-oss-tuning-sdk-test/distillation-val-openai-opposites.jsonlzDgs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test-folder)r'   base_teacher_modelr&   r1   
output_urihttp_optionstest_tune_oss_sftzigs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test-folder/postprocess/node-0/checkpoints/finalz=gs://nathreya-oss-tuning-sdk-test/ayushagra-distillation-test)r&   r1   custom_base_modelr<   r=   test_tune_oss_sft_hyperparamsgMb0?TUNING_MODE_FULL)r&   r1   learning_ratetuning_moder?   r<   r=   test_tune_oss_distillation)r'   r;   r&   r1   r?   r<   r=   &test_tune_oss_distillation_hyperparamszqwen/qwen3@qwen3-4bz!deepseek-ai/deepseek-r1-0528-maasg-C6?   )r'   r;   rB   
batch_sizer<   rC   r=   test_tune_encryption_speczZprojects/vertex-sdk-dev/locations/us-central1/keyRings/ayush-test-key/cryptoKeys/ayush-key)kms_key_name)encryption_spectest_tune_reinforcementREINFORCEMENT_TUNINGd   i   
test-model)autorater_model)autorater_config)autorater_scorer)r%   r&   r'   r0   r/   rG   samples_per_promptevaluate_intervalcheckpoint_intervalmax_output_tokensreward_config!test_tune_reinforcement_composite)weightrV   )weighted_reward_configs)r%   r&   r'   r0   r/   composite_reward_config
test_tableztunings.tune)fileglobals_for_filetest_methodr[   )pytest_asynciozconfig.getoption('--private')zSkipping in pre-public tests)reasonc                 C   sd   | j jr.| jjdtjddtjddddtjddtd	d
}|j	tj
jks'J |jtks0J dS dS zTests tuning with eval config.r   r   r   tuning job with eval configr#   r-   r.   [gs://cloud-samples-data/ai-platform/generative_ai/gemini-2_0/text/sft_validation_data.jsonlr%   r&   r/   r0   r1   evaluation_configr    N)_api_clientvertexaituningstunegenai_typesTuningDatasetCreateTuningJobConfigTuningValidationDatasetre   stateJobStateJOB_STATE_PENDINGclient
tuning_job rt   b/home/djax/ivt_ai_plugin/venv/lib/python3.10/site-packages/google/genai/tests/tunings/test_tune.pytest_eval_configb  s&   
rv   c                 C   s   | j jrNtjtjdddddtjddtjddgtjtjd	d
dtjdddd}| jj	dtj
ddtjddddtjdd|dd}|jtjjksPJ dS dS )&Tests tuning with eval config metrics.prompt-relevanceXHow well does the response address the prompt?: PROMPT: {request}
 RESPONSE: {response}
T:You are a cat. Make all evaluations from this perspective.r
   r   return_raw_outputjudge_model_system_instructionr	   )r
   rouge_1r   r   r   r#   rN   sampling_countrO   r   r   rP   r   r   r   rb   r-   r.   rc   rd   r    N)rf   rg   rj   EvaluationConfigMetricOutputConfigGcsDestinationAutoraterConfigrh   ri   rk   rl   rm   rn   ro   rp   rr   re   rs   rt   rt   ru   test_eval_config_with_metrics}  sL   



$r   c                 C   s   | j jrctjtjtjddtjddddtjtjdddtjtjd	d
dgtj	tj
dddtjdddtjdddd}| jjdtjddtjddddtjdd|dd}|jtjjkseJ dS dS ) rw   ry   rz   T)r|   )metric_prompt_templatesystem_instructioncustom_output_format_config)pointwise_metric_spec)use_effective_order)	bleu_specrouge1)
rouge_type)
rouge_specr   r   r   r#   rN   r   r)   i   )temperaturerU   )r   r   rP   inference_generation_configr   r   r   rb   r-   r.   rc   rd   r    N)rf   rg   rj   r   UnifiedMetricPointwiseMetricSpecCustomOutputFormatConfigBleuSpec	RougeSpecr   r   r   GenerationConfigrh   ri   rk   rl   rm   rn   ro   rp   r   rt   rt   ru   %test_eval_config_with_unified_metrics  sb   

%
4r   c                 C   s   | j jrMtjdddddddiddid	d
diigtjtjdddtjdddd}| jjdtj	ddtj
ddddtjdd|dd}|jtjjksOJ dS dS )rw   rx   ry   Trz   r{   r
   r	   r~   r   r   r   r   r   r#   rN   r   r   r   r   r   rb   r-   r.   rc   rd   r    N)rf   rg   rj   r   r   r   r   rh   ri   rk   rl   rm   rn   ro   rp   r   rt   rt   ru   "test_eval_config_with_metrics_dict  sR   
)r   c                    s`   | j jr,| jjjdtjddtjddddtjddt	d	d
I dH }|j
tjjks.J dS dS ra   )rf   rg   aiorh   ri   rj   rk   rl   rm   re   rn   ro   rp   rq   rt   rt   ru   test_eval_config_async  s&   
r   ))__doc__ r   rj   r   pytestVERTEX_HTTP_OPTIONSr   r   r   r   re   TestTableItemCreateTuningJobParametersrk   rl   TuningMethodr$   rm   EncryptionSpec%SingleReinforcementTuningRewardConfig"ReinforcementTuningAutoraterScorerr   (CompositeReinforcementTuningRewardConfig<CompositeReinforcementTuningRewardConfigWeightedRewardConfigr[   list__annotations__setup__file__globals
pytestmarkpytest_pluginsmarkskipifrv   r   r   r   asyncior   rt   rt   rt   ru   <module>   s  

		
    2

)
9
/