Java: MultiDataSource 导出 CVAuto 数据集新增支持按 ratio 参数对应测试集比例切分数据，优化目录结构

Author: TommyLemon

APIJSON-Java-Server/APIJSONBoot-MultiDataSource/src/main/java/apijson/DatasetUtil.java

@@ -68,7 +68,7 @@ public static void main(String[] args) {
             System.out.println("\nGenerating dataset from JSONObject data...");
             List<JSONObject> sampleData = createSampleJSONObjectData();
             Set<TaskType> jsonTasks = new HashSet<>(Collections.singletonList(TaskType.DETECTION));
-            generate("./output/json_dataset", jsonTasks, sampleData);
+            generate(sampleData, jsonTasks, "./output/detection_dataset/", "train");
 
         } catch (IOException e) {
             e.printStackTrace();
@@ -513,12 +513,13 @@ public static void generate(String outputDir, Set<TaskType> tasks) throws IOExce
 
     /**
      * 从 List<JSONObject> 数据生成 COCO 数据集
-     * @param outputDir 输出目录
-     * @param tasks 任务类型集合
      * @param data 包含图片和标注信息的 JSONObject 列表
+     * @param tasks 任务类型集合
+     * @param outputDir 输出目录
+     * @param part train/val
      * @throws IOException
      */
-    public static void generate(String outputDir, Set<TaskType> tasks, List<JSONObject> data) throws IOException {
+    public static void generate(List<JSONObject> data, Set<TaskType> tasks, String outputDir, String part) throws IOException {
         if (data == null || data.isEmpty()) {
             throw new IllegalArgumentException("Data list cannot be null or empty");
         }
@@ -605,13 +606,15 @@ public static void generate(String outputDir, Set<TaskType> tasks, List<JSONObje
         CocoDataset cocoDataset = builder.build();
 
         // 为不同任务生成不同的文件名
-        String taskName = tasks.iterator().next().toString().toLowerCase();
-        String outputJsonPath = Paths.get(outputDir, "annotations", "instances_" + taskName + ".json").toString();
+        String[] keys = StringUtil.split(outputDir, "/");
+        String taskName = StringUtil.isNotEmpty(part) ? part : (keys != null && keys.length >= 1 ? keys[keys.length - 1] : tasks.iterator().next().toString().toLowerCase());
 
+        String outputJsonPath = Paths.get(outputDir, "annotations", taskName + ".json").toString();
         writeToFile(cocoDataset, outputJsonPath);
+        writeToFile(cocoDataset, Paths.get(outputDir, taskName + ".json").toString());
 
-        // 复制图片文件到指定目录 outputDir/images/
-        copyImagesToDirectory(cocoDataset.getImages(), outputDir + "/images/");
+        // 复制图片文件到指定目录 outputDir/images/ train/val
+        copyImagesToDirectory(cocoDataset.getImages(), outputDir + taskName + "/");
 
         System.out.println("Successfully generated dataset from JSONObject data at: " + outputDir);
     }

APIJSON-Java-Server/APIJSONBoot-MultiDataSource/src/main/java/apijson/boot/FileController.java

@@ -456,25 +456,29 @@ public ResponseEntity<Object> downloadCVReport(@PathVariable(name = "id") String
 	public ResponseEntity<Object> downloadDataset(
 			@PathVariable("id") String idStr,
 			@RequestParam(name = "type", required = false) String type,
-			@RequestParam(name = "datasetName", required = false) String datasetName
+			@RequestParam(name = "ratio", required = false) String ratioStr,
+			@RequestParam(name = "name", required = false) String datasetName
 	) throws IOException {
 		try {
-			// 参数验证
+			long repOrDocId = Long.parseLong(idStr);
+			if (repOrDocId <= 0) {
+				throw new IllegalArgumentException("id 必须为 > 0 的 reportId 或 documentId 有效整数！");
+			}
+
 			if (StringUtil.isNotEmpty(type)) {
 				validateCocoType(type);
 			} else {
 				type = "";
 			}
 
-			String dataset = StringUtil.isEmpty(datasetName) ? type + "_dataset" : datasetName;
-			String exportDir = fileUploadRootDir + dataset + "/";
-
-			long repOrDocId = Long.parseLong(idStr);
-			if (repOrDocId <= 0) {
-				throw new IllegalArgumentException("id 必须为 > 0 的 reportId 或 documentId 有效整数！");
+			int ratio = StringUtil.isEmpty(ratioStr) ? 20 : Integer.parseInt(ratioStr);
+			if (ratio < 0 || ratio > 100) {
+				throw new IllegalArgumentException("测试集比例 ratio 必须为 0 ~ 100 范围内的有效整数！");
 			}
 
-			String name = "CVAuto_" + dataset + repOrDocId + ".zip";
+			String dataset = StringUtil.isNotEmpty(datasetName) ? datasetName : "CVAuto_" + (StringUtil.isNotEmpty(type) ? type + "_" : "") + "dataset_" + repOrDocId;
+			String exportDir = fileUploadRootDir + dataset + "/";
+			String name = dataset + ".zip";
 			String path = fileUploadRootDir + name;
 
 			File file = new File(path);
@@ -499,8 +503,8 @@ public ResponseEntity<Object> downloadDataset(
 
 				{   // [] <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
 					JSONObject item = new JSONObject();
-					//item.put("count", 0);
-					item.put("count", 3);
+					item.put("count", 0);
+					//item.put("count", 3);
 					item.put("join", "@/TestRecord");
 
 					{   // Random <<<<<<<<<<<<<<<<<<<<<<<<<<<<<
@@ -545,17 +549,29 @@ public ResponseEntity<Object> downloadDataset(
 				//long documentId = lastTr == null ? 0 : lastTr.getLongValue("documentId");
 				//long randomId = lastTr == null ? 0 : lastTr.getLongValue("randomId");
 				if (reportId != repOrDocId) {
-					name = "CVAuto_" + dataset + (reportId > 0 ? reportId : repOrDocId + "_last") + ".zip";
+					dataset = StringUtil.isNotEmpty(datasetName) ? datasetName : "CVAuto_" + (StringUtil.isNotEmpty(type) ? type + "_" : "") + "dataset_" + (reportId > 0 ? reportId : repOrDocId + "_last");
+					exportDir = fileUploadRootDir + dataset + "/";
+					name = dataset + ".zip";
 					path = fileUploadRootDir + name;
 				}
 
 				JSONArray array = response.getJSONArray("[]");
 
-				List<JSONObject> list = new ArrayList<>();
+				List<JSONObject> trainList = new ArrayList<>();
+				List<JSONObject> validList = new ArrayList<>();
 				if (array != null) {
-					for (int i = 0; i < array.size(); i++) {
+					int len = array.size();
+					for (int i = 0; i < len; i++) {
 						JSONObject item = array.getJSONObject(i);
-						list.add(item);
+						if (item == null || item.isEmpty()) {
+							continue;
+						}
+
+						if (ratio <= 0 || ratio <= 100 - 100.0*i/len) {
+							trainList.add(item);
+						} else {
+							validList.add(item);
+						}
 
 						//JSONObject random = item == null ? null : item.getJSONObject("Random");
 						//JSONObject testRecord = item == null ? null : item.getJSONObject("TestRecord");
@@ -586,7 +602,9 @@ public ResponseEntity<Object> downloadDataset(
 				//generateCocoDatasetFromApiJson(exportDir, type, dataset, list);
 
 				Set<DatasetUtil.TaskType> detectionTasks = new HashSet<>(Collections.singletonList(DatasetUtil.TaskType.DETECTION));
-				DatasetUtil.generate(exportDir, detectionTasks, list);
+
+				DatasetUtil.generate(trainList, detectionTasks, exportDir, "train");
+				DatasetUtil.generate(validList, detectionTasks, exportDir, "val");
 
 				createZipFromDirectory(exportDir, path);
 

APIJSON-Java-Server/APIJSONBoot-MultiDataSource/src/main/resources/static/cv/js/main.js

@@ -2949,7 +2949,7 @@ https://github.com/Tencent/APIJSON/issues
           }
           else if (this.isRandomShow && this.isRandomListShow) {
             var id = this.reportId || this.getCurrentDocumentId()
-            window.open(this.server + '/download/dataset/' + id)
+            window.open(this.server + '/download/dataset/' + id + "?ratio=20")
           }
           else if (this.view == 'markdown' || this.view == 'output') { //model
             var clazz = StringUtil.trim(this.exTxt.name)