Integry/IntegryManagementSystem
19
0
Fork 0
Code Actions Packages Projects Releases Activity

Implementata delete di file orfani di MinIO sfruttando le API disponibili

Browse Source
This commit is contained in:
Giuseppe Scorrano
2024-09-03 12:21:05 +02:00
parent 40b49300dc
commit 8578e2fa7c
4 changed files with 42 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
ems-core/src/main/java/it/integry/ems/object_storage/interfaces/IObjectStorageService.java
View File

@@ -37,6 +37,4 @@ public interface IObjectStorageService {
boolean isEnabled();
List<String> listFiles(@Nonnull MultiDBTransactionManager multiDBTransactionManager) throws Exception;
boolean bulkDelete(@Nonnull MultiDBTransactionManager multiDBTransactionManager, @Nonnull List<String> objectNameList) throws Exception;
}

64
ems-core/src/main/java/it/integry/ems/object_storage/minio/MinIOService.java
View File

@@ -1,19 +1,17 @@
package it.integry.ems.object_storage.minio;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.io.ByteStreams;
import io.minio.*;
import io.minio.errors.*;
import io.minio.messages.DeleteError;
import io.minio.messages.DeleteObject;
import io.minio.messages.Item;
import it.integry.annotations.PostContextConstruct;
import it.integry.ems.exception.PrimaryDatabaseNotPresentException;
import it.integry.ems.object_storage.interfaces.IObjectStorageService;
import it.integry.ems.service.HttpRestWrapper;
import it.integry.ems.settings.Model.MinIOSettingsModel;
import it.integry.ems.settings.Model.SettingsModel;
import it.integry.ems.sync.MultiDBTransaction.MultiDBTransactionManager;
import org.apache.http.entity.ContentType;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
@@ -24,13 +22,11 @@ import javax.annotation.Nonnull;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -241,11 +237,16 @@ public class MinIOService implements IObjectStorageService {
.map(DeleteObject::new)
.collect(Collectors.toList());
this.mMinioClient.removeObjects(RemoveObjectsArgs.builder()
final Iterable<Result<DeleteError>> results = this.mMinioClient.removeObjects(RemoveObjectsArgs.builder()
.bucket(getBucketName(multiDBTransactionManager))
.objects(objectsToDelete)
.build());
for (final Result<DeleteError> result : results) {
final DeleteError deleteError = result.get();
deleteError.code();
}
return true;
}
@@ -259,50 +260,19 @@ public class MinIOService implements IObjectStorageService {
this.checkStatus();
this.checkConnection();
String url = String.format("http://%s/scandir.php", settingsModel.getMinioConfiguration().getHost());
StringBuilder srb = new StringBuilder();
final Iterable<Result<Item>> results = this.mMinioClient.listObjects(ListObjectsArgs.builder()
.bucket(getBucketName(multiDBTransactionManager))
.build());
HashMap<String, String> queryParam = new HashMap<>();
queryParam.put("bucket", getBucketName(multiDBTransactionManager));
ArrayList<String> files = new ArrayList<>();
int statusResponse = HttpRestWrapper.callGenericGet(url, null, null, srb, queryParam);
if (statusResponse == 200) {
for (final Result<Item> result : results) {
final Item item = result.get();
files.add(item.objectName());
}
ObjectMapper mapper = new ObjectMapper();
JsonNode jsonNode = mapper.readTree(srb.toString());
ArrayList<String> files = new ArrayList<>(jsonNode.size());
jsonNode.forEach(x -> {
files.add(x.asText().replaceAll("\"", ""));
});
return files;
} else return null;
}
@Override
public boolean bulkDelete(@Nonnull MultiDBTransactionManager multiDBTransactionManager, @Nonnull List<String> objectNameList) throws Exception {
this.checkStatus();
this.checkConnection();
String url = String.format("http://%s/deleteBucketContent.php", settingsModel.getMinioConfiguration().getHost());
StringBuilder srb = new StringBuilder();
HashMap<String, String> queryParam = new HashMap<>();
queryParam.put("bucket", getBucketName(multiDBTransactionManager));
ObjectMapper mapper = new ObjectMapper();
StringWriter writer = new StringWriter();
mapper.writeValue(writer, objectNameList);
String dtoJson = writer.toString();
int statusResponse = HttpRestWrapper.callGeneric(url, null, null, dtoJson, ContentType.APPLICATION_JSON, srb, queryParam);
if (statusResponse == 200) {
return true;
} else return false;
return files;
}
private void checkStatus() throws MinIONotEnabledException {

14
ems-engine/src/main/java/it/integry/ems/system/controller/SystemController.java
View File

@@ -1387,15 +1387,9 @@ public class SystemController {
@RequestMapping(value = EmsRestConstants.PATH_OBJECT_STORAGE_ORPHAN_FILES_CLEANER, method = RequestMethod.POST)
public ServiceRestResponse objectStorageOrphanFilesCleaner(HttpServletRequest request,
@RequestParam(CommonConstants.PROFILE_DB) String configuration) {
try {
systemService.minioOrphanFilesCleaner();
return ServiceRestResponse.createPositiveResponse();
} catch (Exception ex) {
logger.error(request.getRequestURI(), ex);
return new ServiceRestResponse(EsitoType.KO, configuration, ex);
}
@RequestParam(CommonConstants.PROFILE_DB) String configuration) throws Exception {
systemService.minioOrphanFilesCleaner();
return ServiceRestResponse.createPositiveResponse();
}
@RequestMapping(value = EmsRestConstants.PATH_DEFAULT_AZIENDA, method = RequestMethod.GET)
@@ -1693,7 +1687,7 @@ public class SystemController {
final List<MtbColt> mtbColtsToFix = UtilityDB.executeSimpleQueryDTO(multiDBTransactionManager.getPrimaryConnection(), queryRetrieveColli, MtbColt.class);
if(mtbColtsToFix == null || mtbColtsToFix.isEmpty())
if (mtbColtsToFix == null || mtbColtsToFix.isEmpty())
return ServiceRestResponse.createPositiveResponse();
for (MtbColt mtbColtToFix : mtbColtsToFix) {

40
ems-engine/src/main/java/it/integry/ems/system/service/SystemService.java
View File

@@ -945,26 +945,28 @@ public class SystemService {
}
}
boolean bulkDelete = setupGest.getSetupBoolean("DATI_AZIENDA", "MINIO", "ENABLE_BULK_DELETE");
this.minIOService.deleteObjects(toDelete, multiDBTransactionManager);
if (bulkDelete)
minIOService.bulkDelete(multiDBTransactionManager, toDelete);
else {
for (int i = 0; i < toDelete.size(); i++) {
if (executor.getQueue().size() > (cores * 4)) {
Thread.sleep(500);
i--;
continue;
}
MinioOrphanTaskDeleter task = new MinioOrphanTaskDeleter(toDelete.get(i), minIOService, multiDBTransactionManager);
executor.execute(task);
}
executor.shutdown();
executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
}
// boolean bulkDelete = setupGest.getSetupBoolean("DATI_AZIENDA", "MINIO", "ENABLE_BULK_DELETE");
//
// if (bulkDelete)
// minIOService.bulkDelete(multiDBTransactionManager, toDelete);
// else {
//
// for (int i = 0; i < toDelete.size(); i++) {
// if (executor.getQueue().size() > (cores * 4)) {
// Thread.sleep(500);
// i--;
// continue;
// }
//
// MinioOrphanTaskDeleter task = new MinioOrphanTaskDeleter(toDelete.get(i), minIOService, multiDBTransactionManager);
// executor.execute(task);
// }
//
// executor.shutdown();
// executor.awaitTermination(Long.MAX_VALUE, TimeUnit.NANOSECONDS);
// }
}
public Object implodeWordDoc() throws Docx4JException {