Coverage for ocp_resources/lm_eval_job.py: 0%

1# Generated using https://github.com/RedHatQE/openshift-python-wrapper/blob/main/scripts/resource/README.md 

2 

3from typing import Any, Dict, List, Optional 

4from ocp_resources.resource import NamespacedResource, MissingRequiredArgumentError 

5 

6 

7class LMEvalJob(NamespacedResource): 

8 """ 

9 LMEvalJob is the Schema for the lmevaljobs API 

10 """ 

11 

12 api_group: str = NamespacedResource.ApiGroup.TRUSTYAI_OPENDATAHUB_IO 

13 

14 def __init__( 

15 self, 

16 allow_code_execution: Optional[bool] = None, 

17 allow_online: Optional[bool] = None, 

18 batch_size: Optional[str] = "", 

19 gen_args: Optional[List[Any]] = None, 

20 limit: Optional[str] = "", 

21 log_samples: Optional[bool] = None, 

22 model: Optional[str] = "", 

23 model_args: Optional[List[Any]] = None, 

24 num_few_shot: Optional[int] = None, 

25 offline: Optional[Dict[str, Any]] = None, 

26 outputs: Optional[Dict[str, Any]] = None, 

27 pod: Optional[Dict[str, Any]] = None, 

28 suspend: Optional[bool] = None, 

29 task_list: Optional[Dict[str, Any]] = None, 

30 **kwargs: Any, 

31 ) -> None: 

32 """ 

33 Args: 

34 allow_code_execution (bool): AllowCodeExecution specifies whether the LMEvalJob can execute remote 

35 code. Default is false. 

36 

37 allow_online (bool): AllowOnly specifies whether the LMEvalJob can directly download remote 

38 code, datasets and metrics. Default is false. 

39 

40 batch_size (str): Batch size for the evaluation. This is used by the models that run and 

41 are loaded locally and not apply for the commercial APIs. 

42 

43 gen_args (List[Any]): Map to `--gen_kwargs` parameter for the underlying library. 

44 

45 limit (str): Accepts an integer, or a float between 0.0 and 1.0 . If passed, will 

46 limit the number of documents to evaluate to the first X documents 

47 (if an integer) per task or first X% of documents per task 

48 

49 log_samples (bool): If this flag is passed, then the model's outputs, and the text fed 

50 into the model, will be saved at per-document granularity 

51 

52 model (str): Model name 

53 

54 model_args (List[Any]): Args for the model 

55 

56 num_few_shot (int): Sets the number of few-shot examples to place in context 

57 

58 offline (Dict[str, Any]): Offline specifies settings for running LMEvalJobs in an offline mode 

59 

60 outputs (Dict[str, Any]): Outputs specifies storage for evaluation results 

61 

62 pod (Dict[str, Any]): Specify extra information for the lm-eval job's pod 

63 

64 suspend (bool): Suspend keeps the job but without pods. This is intended to be used by 

65 the Kueue integration 

66 

67 task_list (Dict[str, Any]): Evaluation task list 

68 

69 """ 

70 super().__init__(**kwargs) 

71 

72 self.allow_code_execution = allow_code_execution 

73 self.allow_online = allow_online 

74 self.batch_size = batch_size 

75 self.gen_args = gen_args 

76 self.limit = limit 

77 self.log_samples = log_samples 

78 self.model = model 

79 self.model_args = model_args 

80 self.num_few_shot = num_few_shot 

81 self.offline = offline 

82 self.outputs = outputs 

83 self.pod = pod 

84 self.suspend = suspend 

85 self.task_list = task_list 

86 

87 def to_dict(self) -> None: 

88 super().to_dict() 

89 

90 if not self.kind_dict and not self.yaml_file: 

91 if not self.model: 

92 raise MissingRequiredArgumentError(argument="self.model") 

93 

94 if not self.task_list: 

95 raise MissingRequiredArgumentError(argument="self.task_list") 

96 

97 self.res["spec"] = {} 

98 _spec = self.res["spec"] 

99 

100 _spec["model"] = self.model 

101 _spec["taskList"] = self.task_list 

102 

103 if self.allow_code_execution is not None: 

104 _spec["allowCodeExecution"] = self.allow_code_execution 

105 

106 if self.allow_online is not None: 

107 _spec["allowOnline"] = self.allow_online 

108 

109 if self.batch_size: 

110 _spec["batchSize"] = self.batch_size 

111 

112 if self.gen_args: 

113 _spec["genArgs"] = self.gen_args 

114 

115 if self.limit: 

116 _spec["limit"] = self.limit 

117 

118 if self.log_samples is not None: 

119 _spec["logSamples"] = self.log_samples 

120 

121 if self.model_args: 

122 _spec["modelArgs"] = self.model_args 

123 

124 if self.num_few_shot: 

125 _spec["numFewShot"] = self.num_few_shot 

126 

127 if self.offline: 

128 _spec["offline"] = self.offline 

129 

130 if self.outputs: 

131 _spec["outputs"] = self.outputs 

132 

133 if self.pod: 

134 _spec["pod"] = self.pod 

135 

136 if self.suspend is not None: 

137 _spec["suspend"] = self.suspend 

138 

139 # End of generated code