Integry/IntegryManagementSystem
19
0
Fork 0
Code Actions Packages Projects Releases Activity

Merge remote-tracking branch 'origin/develop' into develop

Browse Source
This commit is contained in:
Mina Ruggiero
2024-02-28 17:49:06 +01:00
parent a2d536d6d5 507aa7cd0d
commit bfbc459393
23 changed files with 824 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

59
IntegryManagementSystem.ipr
View File

@@ -108,6 +108,9 @@
<element id="library" level="project" name="Maven: com.fasterxml.jackson.core:jackson-core:2.9.4" />
<element id="library" level="project" name="Maven: com.fasterxml.jackson.core:jackson-databind:2.9.4" />
<element id="library" level="project" name="Maven: com.fasterxml.jackson.dataformat:jackson-dataformat-csv:2.9.4" />
<element id="library" level="project" name="Maven: com.alibaba:fastjson:2.0.47" />
<element id="library" level="project" name="Maven: com.alibaba.fastjson2:fastjson2-extension:2.0.47" />
<element id="library" level="project" name="Maven: com.alibaba.fastjson2:fastjson2:2.0.47" />
<element id="library" level="project" name="Maven: org.springframework:spring-aop:4.2.5.RELEASE" />
<element id="library" level="project" name="Maven: org.springframework:spring-core:4.2.5.RELEASE" />
<element id="library" level="project" name="Maven: org.springframework:spring-instrument-tomcat:4.2.5.RELEASE" />
@@ -254,7 +257,7 @@
<element id="library" level="project" name="Maven: org.reflections:reflections:0.9.10" />
<element id="library" level="project" name="Maven: org.javassist:javassist:3.19.0-GA" />
<element id="library" level="project" name="Maven: com.google.code.findbugs:annotations:2.0.1" />
<element id="library" level="project" name="Maven: com.google.code.gson:gson:2.5" />
<element id="library" level="project" name="Maven: com.google.code.gson:gson:2.9.1" />
<element id="library" level="project" name="Maven: org.apache.httpcomponents:httpclient:4.3.6" />
<element id="library" level="project" name="Maven: org.apache.httpcomponents:httpcore:4.3.3" />
<element id="library" level="project" name="Maven: commons-logging:commons-logging:1.1.3" />
@@ -721,7 +724,7 @@
</component>
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Tomcat " type="#com.intellij.j2ee.web.tomcat.TomcatRunConfigurationFactory" factoryName="Local" APPLICATION_SERVER_NAME="Tomcat" ALTERNATIVE_JRE_ENABLED="true" ALTERNATIVE_JRE_PATH="1.8" show_console_on_std_err="true" show_console_on_std_out="true">
<option name="COMMON_VM_ARGUMENTS" value="-Xms256m -Xmx1024m -XX:PermSize=1024m" />
<option name="COMMON_VM_ARGUMENTS" value="-Xms512m -Xmx2048m" />
<deployment>
<artifact name="ems-engine:war exploded">
<settings>
@@ -867,7 +870,7 @@
<method v="2" />
</configuration>
<configuration default="false" name="Tomcat (FAST)" type="#com.intellij.j2ee.web.tomcat.TomcatRunConfigurationFactory" factoryName="Local" APPLICATION_SERVER_NAME="Tomcat" ALTERNATIVE_JRE_ENABLED="false" ALTERNATIVE_JRE_PATH="1.8" show_console_on_std_err="true" show_console_on_std_out="true">
<option name="COMMON_VM_ARGUMENTS" value="-DDISABLE_DROOLS_COMPILE=TRUE -Xms256m -Xmx1024m -XX:PermSize=1024m" />
<option name="COMMON_VM_ARGUMENTS" value="-DDISABLE_DROOLS_COMPILE=TRUE -Xms512m -Xmx2048m" />
<deployment>
<artifact name="ems-engine:war exploded">
<settings>
@@ -937,7 +940,7 @@
<option name="PROGRAM_PARAMETERS" value="" />
</SHUTDOWN>
</ConfigurationWrapper>
<ConfigurationWrapper VM_VAR="JAVA_OPTS" RunnerId="JProfiler">
<ConfigurationWrapper VM_VAR="JAVA_OPTS" RunnerId="Profile">
<option name="USE_ENV_VARIABLES" value="true" />
<STARTUP>
<option name="USE_DEFAULT" value="true" />
@@ -952,7 +955,7 @@
<option name="PROGRAM_PARAMETERS" value="" />
</SHUTDOWN>
</ConfigurationWrapper>
<ConfigurationWrapper VM_VAR="JAVA_OPTS" RunnerId="Profile">
<ConfigurationWrapper VM_VAR="JAVA_OPTS" RunnerId="Profile ">
<option name="USE_ENV_VARIABLES" value="true" />
<STARTUP>
<option name="USE_DEFAULT" value="true" />
@@ -1117,6 +1120,42 @@
<root url="jar://$MAVEN_REPOSITORY$/c3p0/c3p0/0.9.1.1/c3p0-0.9.1.1-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: com.alibaba.fastjson2:fastjson2-extension:2.0.47" type="java-imported" external-system-id="Maven">
<properties groupId="com.alibaba.fastjson2" artifactId="fastjson2-extension" version="2.0.47" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson2/fastjson2-extension/2.0.47/fastjson2-extension-2.0.47.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson2/fastjson2-extension/2.0.47/fastjson2-extension-2.0.47-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson2/fastjson2-extension/2.0.47/fastjson2-extension-2.0.47-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: com.alibaba.fastjson2:fastjson2:2.0.47" type="java-imported" external-system-id="Maven">
<properties groupId="com.alibaba.fastjson2" artifactId="fastjson2" version="2.0.47" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson2/fastjson2/2.0.47/fastjson2-2.0.47.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson2/fastjson2/2.0.47/fastjson2-2.0.47-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson2/fastjson2/2.0.47/fastjson2-2.0.47-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: com.alibaba:fastjson:2.0.47" type="java-imported" external-system-id="Maven">
<properties groupId="com.alibaba" artifactId="fastjson" version="2.0.47" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson/2.0.47/fastjson-2.0.47.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson/2.0.47/fastjson-2.0.47-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/alibaba/fastjson/2.0.47/fastjson-2.0.47-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: com.amazonservices.mws:MWSClientJavaRuntime:1.1" type="java-imported" external-system-id="Maven">
<properties groupId="com.amazonservices.mws" artifactId="MWSClientJavaRuntime" version="1.1" />
<CLASSES>
@@ -1501,16 +1540,16 @@
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: com.google.code.gson:gson:2.5" type="java-imported" external-system-id="Maven">
<properties groupId="com.google.code.gson" artifactId="gson" version="2.5" />
<library name="Maven: com.google.code.gson:gson:2.9.1" type="java-imported" external-system-id="Maven">
<properties groupId="com.google.code.gson" artifactId="gson" version="2.9.1" />
<CLASSES>
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/gson/gson/2.5/gson-2.5.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/gson/gson/2.9.1/gson-2.9.1.jar!/" />
</CLASSES>
<JAVADOC>
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/gson/gson/2.5/gson-2.5-javadoc.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/gson/gson/2.9.1/gson-2.9.1-javadoc.jar!/" />
</JAVADOC>
<SOURCES>
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/gson/gson/2.5/gson-2.5-sources.jar!/" />
<root url="jar://$MAVEN_REPOSITORY$/com/google/code/gson/gson/2.9.1/gson-2.9.1-sources.jar!/" />
</SOURCES>
</library>
<library name="Maven: com.google.errorprone:error_prone_annotations:2.0.18" type="java-imported" external-system-id="Maven">

6
ems-core/pom.xml
View File

@@ -437,6 +437,12 @@
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>2.0.47</version>
</dependency>
<!-- Spring -->
<dependency>
<groupId>org.springframework</groupId>

9
ems-core/src/main/java/it/integry/ems/adapter/BooleanDeserializer.java
View File

@@ -37,17 +37,10 @@ public class BooleanDeserializer extends JsonDeserializer<Boolean> {
return Boolean.FALSE;
}
// if ("N".equalsIgnoreCase(text) || text.length() == 0) {
// return Boolean.FALSE;
// }
//
// if ("Y".equalsIgnoreCase(text)) {
// return Boolean.TRUE;
// }
throw ctxt.weirdStringException("", _valueClass, "only \"true\" or \"false\" recognized");
}
// Otherwise, no can do:
throw ctxt.mappingException(_valueClass);
throw ctxt.instantiationException(_valueClass, "error");
}
}

2
ems-core/src/main/java/it/integry/ems/controller/EmsController.java
View File

@@ -705,7 +705,7 @@ public class EmsController {
if (ordinaSuPriorita)
entityList = entityPropertyHolder.getEntityChain(entityList, multiDBTransactionManager.getPrimaryConnection());
Integer prevTransactionGroupId = null;
Long prevTransactionGroupId = null;
for (EntityBase entity : entityList) {
entity.setTransactionGroupId(prevTransactionGroupId);

18
ems-core/src/main/java/it/integry/ems/rules/businessLogic/LoadColliService.java
View File

@@ -27,6 +27,8 @@ import java.sql.ResultSet;
import java.text.SimpleDateFormat;
import java.util.*;
import static it.integry.ems.rules.completing.QueryRules.getSingleValue;
@Service
@Scope("request")
public class LoadColliService {
@@ -262,6 +264,16 @@ public class LoadColliService {
dtbDoct.setMtbColt(mtbColt);
}
// Acquisizione ID_LOTTO del primo collo da agganciare
if (loadDatiFromDB) {
query = "SELECT TOP 1 id_lotto FROM mtb_colt ";
query = UtilityDB.addwhereCond(query, whereCond, false);
Integer idLotto = (Integer) getSingleValue(multiDBTransactionManager.getPrimaryConnection(), query);
if (idLotto != null){
dtbDoct.setIdLotto(idLotto);
}
}
Stream.of(loadColli.getColli()).forEach(x -> x.setUpdateColliFromDoc(true));
//AGGIUNGERE CAMPI CHE PASSIAMO DA FUORI
@@ -585,7 +597,8 @@ public class LoadColliService {
" dtb_ordr.cod_promo as 'cod_promo', " +
" dtb_ordr.cod_art_for as 'cod_art_for', " +
" mvw_kit.flag_kit as 'articolo_composto', " +
" IsNull(mvw_colr_sum.cod_jcom, dtb_ordr.cod_jcom) as 'cod_jcom', " +
/*" IsNull(mvw_colr_sum.cod_jcom, dtb_ordr.cod_jcom) as 'cod_jcom', " +*/
" dtb_ordr.cod_jcom as 'cod_jcom', " +
" IsNull(dtb_ordr.unt_ord2, mtb_aart.unt_mis2) as 'unt_doc2', " +
" IsNull(dtb_ordr.unt_ord3, mtb_aart.unt_mis3) as 'unt_doc3', " +
" mtb_aart.tara_kg as 'tara_kg', " +
@@ -652,7 +665,8 @@ public class LoadColliService {
" mtb_aart.flag_stato, " +
" dtb_ordr.cod_art_for, " +
" mvw_kit.flag_kit, " +
" IsNull(mvw_colr_sum.cod_jcom, dtb_ordr.cod_jcom), " +
/*" IsNull(mvw_colr_sum.cod_jcom, dtb_ordr.cod_jcom), " +*/
" dtb_ordr.cod_jcom, " +
" IsNull(dtb_ordr.unt_ord2, mtb_aart.unt_mis2), " +
" IsNull(dtb_ordr.unt_ord3, mtb_aart.unt_mis3), " +
" mtb_aart.tara_kg, " +

2
ems-core/src/main/java/it/integry/ems/service/EntityProcessor.java
View File

@@ -498,7 +498,7 @@ public class EntityProcessor {
entities = entityPropertyHolder.getEntityChain(entities, multiDBTransactionManager.getPrimaryDatasource().getConnection());
}
Integer prevTransactionGroupId = null;
Long prevTransactionGroupId = null;
for (int i = 0; i < entities.size(); i++) {
EntityBase entity = entities.get(i);

227
ems-core/src/main/java/it/integry/ems/sync/AsyncHistoryManager.java
View File

@@ -1,19 +1,25 @@
package it.integry.ems.sync;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement;
import it.integry.annotations.PostContextConstruct;
import it.integry.ems.datasource.DataSource;
import it.integry.ems.json.JSONObjectMapper;
import it.integry.ems.looper.service.LooperService;
import it.integry.ems.sync.MultiDBTransaction.MultiDBTransactionManager;
import it.integry.ems.sync.dto.ExportHistoryGroupDTO;
import it.integry.ems.sync.dto.ExportHistoryItemDTO;
import it.integry.ems.sync.dto.ExportHistoryStatusDTO;
import it.integry.ems.sync.dto.PublicationDTO;
import it.integry.ems_model.base.EntityBase;
import it.integry.ems_model.base.EntityPropertyHolder;
import it.integry.ems_model.entity.StbPublications;
import it.integry.ems_model.entity.StbPublicationsDetail;
import it.integry.ems_model.entity.StbTransactionLog;
import it.integry.ems_model.types.OperationType;
import it.integry.ems_model.utility.UtilityDB;
import it.integry.ems_model.utility.UtilityLocalDate;
import javafx.util.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
@@ -22,7 +28,10 @@ import org.springframework.stereotype.Component;
import java.sql.Connection;
import java.time.LocalDateTime;
import java.time.temporal.ChronoUnit;
import java.util.*;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
@Component
public class AsyncHistoryManager {
@@ -39,27 +48,36 @@ public class AsyncHistoryManager {
private JSONObjectMapper jsonObjectMapper;
//Implement check list like distribuzione
private final HashMap<Long, List<ExportHistoryItemDTO>> currentlyInExecution = new HashMap<>();
private final AtomicReference<List<ExportHistoryGroupDTO>> currentlyInExecutionG = new AtomicReference<>(new ArrayList<>());
@PostContextConstruct
public void init() {
this.looperService.add(this::consumeRetrieveQueue, 10 * 1000, "async-history-read");
this.looperService.add(this::consumeProcessedQueue, 10 * 1000, "async-history-save");
this.looperService.add(this::consumeProcessedQueue, 2 * 1000, "async-history-save");
}
public void addToExportQueue(DataSource dataSource, long groupId, PublicationDTO publication) throws Exception {
tryAddInExecutionList(groupId, new ExportHistoryItemDTO()
.setPublicationGroupId(groupId)
.setPublication(publication)
.setDataSource(dataSource));
tryAddInExecutionList(groupId, dataSource, new ExportHistoryItemDTO()
.setPublication(publication));
}
private void tryAddInExecutionList(long groupId, ExportHistoryItemDTO exportHistoryItemDTO) throws Exception {
currentlyInExecution.putIfAbsent(groupId, new ArrayList<>());
private void tryAddInExecutionList(long groupId, DataSource dataSource, ExportHistoryItemDTO exportHistoryItemDTO) throws Exception {
List<ExportHistoryItemDTO> list = currentlyInExecution.get(groupId);
ExportHistoryGroupDTO currentlyInExec = currentlyInExecutionG.get().stream()
.filter(x -> x.getGroupId() == groupId)
.findFirst().orElseGet(() ->
{
final ExportHistoryGroupDTO exportHistoryGroupDTO = new ExportHistoryGroupDTO()
.setDataSource(dataSource)
.setGroupId(groupId);
currentlyInExecutionG.get().add(exportHistoryGroupDTO);
return exportHistoryGroupDTO;
});
List<ExportHistoryItemDTO> list = currentlyInExec.getItems();
final boolean alreadyRegistered = list.stream()
.anyMatch(x -> x.getPublication().getId() == exportHistoryItemDTO.getPublication().getId());
@@ -67,164 +85,181 @@ public class AsyncHistoryManager {
throw new Exception("Il sistema sta già elaborando questa publication");
}
exportHistoryItemDTO.setTotalItemCount(countExistingItems(exportHistoryItemDTO));
currentlyInExec.setTotalItemCount(currentlyInExec.getTotalItemCount() + countExistingItems(dataSource, exportHistoryItemDTO));
list.add(exportHistoryItemDTO);
}
private void consumeRetrieveQueue() {
final List<Long> currentlyInExecutionKeys = new ArrayList<>(currentlyInExecution.keySet());
for (long currentGroupId : currentlyInExecutionKeys) {
List<ExportHistoryItemDTO> entitiesToExport = currentlyInExecution.get(currentGroupId);
for (ExportHistoryGroupDTO currentGroup : currentlyInExecutionG.get()) {
List<ExportHistoryItemDTO> entitiesToExport = currentGroup.getItems();
if (currentGroup.getStartTime() != null)
continue;
currentGroup.setStartTime(UtilityLocalDate.getNowTime());
for (ExportHistoryItemDTO entityToExport : entitiesToExport) {
try {
internalExportEntity(entityToExport);
internalExportEntity(currentGroup, entityToExport);
} catch (Exception ex) {
ex.printStackTrace();
logger.error(ex.getMessage(), ex);
}
}
currentGroup.setCompletedRead(true);
}
}
private void internalExportEntity(ExportHistoryItemDTO entityHistoryToExport) throws Exception {
private void internalExportEntity(ExportHistoryGroupDTO exportHistoryGroup, ExportHistoryItemDTO entityHistoryToExport) throws Exception {
if (entityHistoryToExport.getStartDate() == null)
entityHistoryToExport.setStartDate(UtilityLocalDate.getNowTime());
final Class<? extends EntityBase> entityClass = entityPropertyHolder.getEntityClassFromTableName(entityHistoryToExport.getPublication().getEntityName());
try (MultiDBTransactionManager newConnection = new MultiDBTransactionManager(entityHistoryToExport.getDataSource().getProfile())) {
retrieveEntitiesByChunk(newConnection.getPrimaryConnection(), entityHistoryToExport, entityClass);
try (MultiDBTransactionManager newConnection = new MultiDBTransactionManager(exportHistoryGroup.getDataSource().getProfile())) {
retrieveEntitiesByChunk(newConnection.getPrimaryConnection(), exportHistoryGroup, entityHistoryToExport, entityClass);
}
}
private long countExistingItems(ExportHistoryItemDTO entityHistoryToExport) throws Exception {
private long countExistingItems(DataSource dataSource, ExportHistoryItemDTO entityHistoryToExport) throws Exception {
final Class<? extends EntityBase> entityClass = entityPropertyHolder.getEntityClassFromTableName(entityHistoryToExport.getPublication().getEntityName());
try (MultiDBTransactionManager newConnection = new MultiDBTransactionManager(entityHistoryToExport.getDataSource().getProfile())) {
try (MultiDBTransactionManager newConnection = new MultiDBTransactionManager(dataSource.getProfile())) {
String sql = "SELECT CONVERT(BIGINT, COUNT(*)) FROM " + entityClass.newInstance().getTableName();
return UtilityDB.executeSimpleQueryOnlyFirstRowFirstColumn(newConnection.getPrimaryConnection(), sql);
}
}
private void retrieveEntitiesByChunk(Connection connection, ExportHistoryItemDTO exportHistoryItem, Class<? extends EntityBase> entityClass) throws Exception {
private void retrieveEntitiesByChunk(Connection connection, ExportHistoryGroupDTO exportHistoryGroup, ExportHistoryItemDTO exportHistoryItem, Class<? extends EntityBase> entityClass) throws Exception {
long totalItemCount = exportHistoryItem.getTotalItemCount();
int chunkSize = 10000;
for (int chunkIndex = 0; chunkIndex < (totalItemCount / chunkSize) + 1; chunkIndex++) {
while (exportHistoryItem.getToProcessQueue().size() > 30000) {
Thread.sleep(10 * 1000);
}
String innerSql = "SELECT *, ROW_NUMBER() OVER (ORDER BY (SELECT NULL)) AS row_number " +
String innerSql = "SELECT * " +
"FROM " + exportHistoryItem.getPublication().getEntityName();
innerSql = UtilityDB.addwhereCond(innerSql, exportHistoryItem.getPublication().getWhereCond(), true);
String sql = "WITH TempResult AS (\n" +
innerSql + ")\n" +
"SELECT TempResult.*\n" +
"FROM TempResult\n" +
"ORDER BY row_number\n" +
"OFFSET (" + chunkIndex + ") * " + chunkSize + " ROWS FETCH NEXT " + chunkSize + " ROWS ONLY";
UtilityDB.executeSimpleQueryDTOAsync(connection, innerSql, entityClass, data -> {
final List<? extends EntityBase> selectedChunk = UtilityDB.executeSimpleQueryDTO(connection, sql, entityClass);
exportHistoryGroup.setProcessedItemCount(exportHistoryGroup.getProcessedItemCount() + 1);
// exportHistoryItem.getToProcessQueue().add(JSON.toJSONString(data));
if (selectedChunk == null || selectedChunk.isEmpty()) continue;
exportHistoryItem.setProcessedItemCount(exportHistoryItem.getProcessedItemCount() + selectedChunk.size());
//logger.debug("COUNT: " + exportHistoryItem.getProcessedItemCount());
selectedChunk.forEach(x -> exportHistoryItem.getToProcessQueue().add(x));
try {
exportHistoryGroup.getToProcessQueue().add(new Pair<>(exportHistoryItem.getPublication().getEntityName(), jsonObjectMapper.writeValueAsString(data)));
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
while ((float) (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / Runtime.getRuntime().totalMemory() > 0.9f) {
logger.debug("Memory limit reached: " + (((float) (Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / Runtime.getRuntime().totalMemory()) * 100) + "%");
try {
Thread.sleep(1 * 1000);
} catch (InterruptedException e) {
throw new RuntimeException(e);
}
}
});
}
private void consumeProcessedQueue() {
final List<Long> currentlyInExecutionKeys = new ArrayList<>(currentlyInExecution.keySet());
for (long currentGroupId : currentlyInExecutionKeys) {
List<ExportHistoryItemDTO> entitiesToExport = currentlyInExecution.get(currentGroupId);
for (ExportHistoryItemDTO entityToExport : entitiesToExport) {
List<ExportHistoryGroupDTO> list = currentlyInExecutionG.get();
for (int i = 0; i < list.size(); i++) {
ExportHistoryGroupDTO currentGroup = list.get(i);
try {
internalProcessQueue(entityToExport);
internalProcessQueue(currentGroup);
if (currentGroup.isCompletedRead() && currentGroup.getSyncronizedItemCount() >= currentGroup.getTotalItemCount() && currentGroup.getEndTime() == null) {
currentGroup.setEndTime(UtilityLocalDate.getNowTime());
updateReadyToTransmit(currentGroup);
list.remove(i--);
}
currentGroup.getDataSource().getConnection().commit();
} catch (Exception ex) {
ex.printStackTrace();
logger.error(ex.getMessage(), ex);
}
}
}
private void internalProcessQueue(ExportHistoryGroupDTO currentGroup) throws Exception {
String insertSQL = "INSERT INTO " + StbTransactionLog.ENTITY + " (publication_group_id, created_at, user_name, entities, entities_json, group_id)" +
" VALUES (?, ?, ?, ?, ?, ?)";
try (MultiDBTransactionManager multiDBTransactionManager = new MultiDBTransactionManager(currentGroup.getDataSource().getProfile())) {
while (!currentGroup.getToProcessQueue().isEmpty()) {
try (SQLServerPreparedStatement insertBulkPs = (SQLServerPreparedStatement) multiDBTransactionManager.prepareStatement(insertSQL)) {
Pair<String, String> entityJson;
while ((entityJson = currentGroup.getToProcessQueue().poll()) != null) {
insertBulkPs.setLong(1, currentGroup.getGroupId());
insertBulkPs.setObject(2, UtilityLocalDate.getNowTime());
insertBulkPs.setString(3, null);
insertBulkPs.setString(4, entityJson.getKey());
insertBulkPs.setString(5, entityJson.getValue());
insertBulkPs.setLong(6, -1);
insertBulkPs.addBatch();
currentGroup.setSyncronizedItemCount(currentGroup.getSyncronizedItemCount() + 1);
}
private void internalProcessQueue(ExportHistoryItemDTO entityToExport) throws Exception {
//logger.debug("TBS COUNT: " + entityToExport.getToProcessQueue().size());
insertBulkPs.executeLargeBatch();
if (entityToExport.getStartDate() == null) entityToExport.setStartDate(UtilityLocalDate.getNowTime());
try (MultiDBTransactionManager multiDBTransactionManager = new MultiDBTransactionManager(entityToExport.getDataSource().getProfile())) {
EntityBase entity;
while ((entity = entityToExport.getToProcessQueue().poll()) != null) {
StbTransactionLog stbTransactionLog = new StbTransactionLog()
.setPublicationGroupId(entityToExport.getPublicationGroupId())
.setCreatedAt(UtilityLocalDate.getNowTime())
.setEntitiesJson(jsonObjectMapper.writeValueAsString(entity))
.setGroupId(-1)
.setEntities(entityToExport.getPublication().getEntityName());
stbTransactionLog
.setOperation(OperationType.INSERT);
stbTransactionLog.manageWithParentConnection(multiDBTransactionManager.getPrimaryConnection());
entityToExport.setSyncronizedItemCount(entityToExport.getSyncronizedItemCount() + 1);
System.gc();
}
}
}
}
private void updateReadyToTransmit(ExportHistoryGroupDTO exportHistoryGroup) throws Exception {
StbPublications stbPublications = new StbPublications()
.setId(exportHistoryGroup.getGroupId());
stbPublications.setOperation(OperationType.NO_OP);
stbPublications.setStbPublicationsDetails(exportHistoryGroup.getItems().stream()
.map(x -> {
StbPublicationsDetail stbPublicationsDetail = new StbPublicationsDetail()
.setId(x.getPublication().getId())
.setReadyToTransmit(true);
stbPublicationsDetail.setOperation(OperationType.UPDATE);
return stbPublicationsDetail;
})
.collect(Collectors.toList()));
stbPublications.manageWithParentConnection(exportHistoryGroup.getDataSource().getConnection());
}
public List<ExportHistoryStatusDTO> getStatus() {
List<ExportHistoryStatusDTO> statusList = new ArrayList<>();
HashMap<Long, HashMap<String, Object>> total = new HashMap<>();
for (ExportHistoryGroupDTO currentGroup : currentlyInExecutionG.get()) {
for (Long key : currentlyInExecution.keySet()) {
final LocalDateTime startDate = currentlyInExecution.get(key).stream()
.map(ExportHistoryItemDTO::getStartDate)
.filter(Objects::nonNull)
.min(Comparator.naturalOrder())
.orElse(null);
final LocalDateTime startDate = currentGroup.getStartTime();
long totalCount = 0;
long processedCount = 0;
long itemsPerSec = 1;
LocalDateTime estimatedEnd = null;
LocalDateTime estimatedEnd = currentGroup.getEndTime();
if (startDate != null) {
totalCount = currentlyInExecution.get(key).stream()
.map(ExportHistoryItemDTO::getTotalItemCount)
.reduce(0L, Long::sum);
totalCount = currentGroup.getTotalItemCount();
processedCount = currentGroup.getSyncronizedItemCount();
processedCount = currentlyInExecution.get(key).stream()
.map(ExportHistoryItemDTO::getSyncronizedItemCount)
.reduce(0L, Long::sum);
final long secondsBetween = ChronoUnit.SECONDS.between(startDate, UtilityLocalDate.getNowTime());
final long secondsBetween = ChronoUnit.SECONDS.between(startDate, estimatedEnd != null ? estimatedEnd : UtilityLocalDate.getNowTime());
itemsPerSec = processedCount / (secondsBetween == 0 ? 1 : secondsBetween);
//if(itemsPerSec == 0) itemsPerSec = 1 / 60;
double secsToEnd = (double) totalCount / (itemsPerSec == 0 ? 1 : itemsPerSec);
estimatedEnd = startDate.plusSeconds((long) secsToEnd);
}
statusList.add(new ExportHistoryStatusDTO()
.setPublicationGroupId(key)
.setPublicationGroupId(currentGroup.getGroupId())
.setStartedAt(startDate)
.setTotalCount(totalCount)
.setProcessedCount(processedCount)

142
ems-core/src/main/java/it/integry/ems/sync/AsyncManager.java
View File

@@ -1,18 +1,21 @@
package it.integry.ems.sync;
import com.microsoft.sqlserver.jdbc.SQLServerPreparedStatement;
import it.integry.annotations.PostContextConstruct;
import it.integry.common.var.CommonConstants;
import it.integry.ems.json.JSONObjectMapper;
import it.integry.ems.looper.service.LooperService;
import it.integry.ems.settings.Model.AvailableConnectionsModel;
import it.integry.ems.settings.Model.SettingsModel;
import it.integry.ems.settings.SettingsController;
import it.integry.ems.sync.MultiDBTransaction.MultiDBTransactionManager;
import it.integry.ems.utility.UtilityDebug;
import it.integry.ems_model.base.EntityBase;
import it.integry.ems_model.entity.StbPublicationsDetail;
import it.integry.ems_model.entity.StbTransactionLog;
import it.integry.ems_model.types.OperationType;
import it.integry.ems_model.utility.UtilityDB;
import it.integry.ems_model.utility.UtilityLocalDate;
import it.integry.ems_model.utility.UtilityString;
import javafx.util.Pair;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.josql.Query;
@@ -21,11 +24,12 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.context.ContextLoader;
import java.io.StringWriter;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.stream.Collectors;
@Component
@@ -45,12 +49,14 @@ public class AsyncManager {
private final static HashMap<String, List<StbPublicationsDetail>> cachedSetup = new HashMap<>();
private final static ConcurrentLinkedQueue<Pair<String, StbTransactionLog>> toBeSavedQueue = new ConcurrentLinkedQueue<>();
@PostContextConstruct
public void init() {
if (!UtilityDebug.isDebugExecution() && !UtilityDebug.isIntegryServer()) {
looperService.add(this::internalCachePublicationsSetup, 5 * 60 * 1000, AsyncManager.class.getName());
}
//if (!UtilityDebug.isDebugExecution() && !UtilityDebug.isIntegryServer()) {
looperService.add(this::internalCachePublicationsSetup, 5 * 60 * 1000, "sync-setup-cache");
looperService.add(this::consumeToBeSavedQueue, 20 * 1000, "sync-flush-data");
//}
}
private void internalCachePublicationsSetup() {
@@ -130,49 +136,91 @@ public class AsyncManager {
}
public static void saveNewTransaction(Connection connection, EntityBase entityBase, int transactionGroupId) throws Exception {
String transactionDate = new SimpleDateFormat(CommonConstants.DATETIME_FORMAT_YMD).format(new Date());
int transactionId = 0;
Integer tmpTransactionId = UtilityDB.executeSimpleQueryOnlyFirstRowFirstColumn(
connection, "SELECT max(transaction_id) as max_id from stb_transaction_log_db");
if (tmpTransactionId != null) transactionId = tmpTransactionId;
transactionId++; //Incremento l'ID
public static Long saveNewTransaction(Connection connection, String dbName, EntityBase entityBase, long publicationId, Long transactionGroupId) throws Exception {
//TODO: Calc transaction group ID here
if (transactionGroupId == null) transactionGroupId = getNextTransactionGroupId(connection, dbName);
JSONObjectMapper jsonObjectMapper = ContextLoader.getCurrentWebApplicationContext().getBean(JSONObjectMapper.class);
StringWriter writer = new StringWriter();
jsonObjectMapper.writeValue(writer, entityBase);
String entityName = entityBase.getTableName().toUpperCase();
String sql =
"INSERT INTO stb_transaction_log_db(transaction_id, transaction_group_id, transaction_date, user_name, transaction_json, entity_name_list) " +
"VALUES (" + transactionId + "," + transactionGroupId + ",'" + transactionDate + "', null, '%s', " + UtilityDB.valueToString(entityName) + ")";
String jsonString = writer.toString().replace("'", "''");
sql = String.format(sql, jsonString);
PreparedStatement psExec = connection.prepareStatement(sql);
psExec.executeUpdate();
psExec.close();
}
public static int getNextTransactionGroupId(Connection connection) throws Exception {
int transactionGroupId = 0;
Integer tmpTransactionGroupId = UtilityDB.executeSimpleQueryOnlyFirstRowFirstColumn(
connection, "SELECT MAX(transaction_group_id) AS max_id from stb_transaction_log_db");
if (tmpTransactionGroupId != null) transactionGroupId = tmpTransactionGroupId;
transactionGroupId++; //Incremento l'ID
StbTransactionLog stbTransactionLog = new StbTransactionLog()
.setCreatedAt(UtilityLocalDate.getNowTime())
.setEntities(entityBase.getTableName().toUpperCase())
.setEntitiesJson(jsonObjectMapper.writeValueAsString(entityBase))
.setPublicationGroupId(publicationId)
.setUserName(entityBase.getUsername())
.setGroupId(transactionGroupId);
stbTransactionLog.setOperation(OperationType.INSERT);
toBeSavedQueue.add(new Pair<>(dbName, stbTransactionLog));
return transactionGroupId;
}
public static long getNextTransactionGroupId(Connection connection, String dbName) throws Exception {
final long maxToBeProcessed = toBeSavedQueue.stream().filter(x -> x.getKey().equalsIgnoreCase(dbName))
.map(x -> x.getValue().getGroupId())
.max(Long::compare)
.orElse(0L);
long transactionGroupId = 0;
if (maxToBeProcessed <= 0) {
Long tmpTransactionGroupId = UtilityDB.executeSimpleQueryOnlyFirstRowFirstColumn(
connection, "SELECT MAX(group_id) AS max_id FROM " + StbTransactionLog.ENTITY);
tmpTransactionGroupId = tmpTransactionGroupId != 0 ? tmpTransactionGroupId : 0L;
transactionGroupId = Math.max(maxToBeProcessed, tmpTransactionGroupId);
} else {
transactionGroupId = maxToBeProcessed;
}
transactionGroupId++; //Incremento l'ID
return transactionGroupId;
}
private void consumeToBeSavedQueue() {
if(cachedSetup.entrySet().stream().anyMatch(x -> x.getValue() != null && x.getValue().stream().anyMatch(y -> !y.isReadyToTransmit())))
return;
List<String> dbNamesToConnect = toBeSavedQueue.stream().map(Pair::getKey).distinct().collect(Collectors.toList());
final List<AvailableConnectionsModel> databaseConnections = settingsModel.getAvailableConnections().stream()
.filter(x -> dbNamesToConnect.stream().anyMatch(y -> x.getDbName().equalsIgnoreCase(y)))
.collect(Collectors.toList());
String insertSQL = "INSERT INTO " + StbTransactionLog.ENTITY + " (publication_group_id, created_at, user_name, entities, entities_json, group_id)" +
" VALUES (?, ?, ?, ?, ?, ?)";
Pair<String, StbTransactionLog> itemToSave = null;
while ((itemToSave = toBeSavedQueue.poll()) != null) {
final Pair<String, StbTransactionLog> finalItemToSave = itemToSave;
final StbTransactionLog stbTransactionLog = finalItemToSave.getValue();
AvailableConnectionsModel connectionModel = databaseConnections.stream()
.filter(x -> x.getDbName().equalsIgnoreCase(finalItemToSave.getKey()))
.findFirst()
.get();
try (MultiDBTransactionManager multiDBTransactionManager = new MultiDBTransactionManager(connectionModel);
SQLServerPreparedStatement insertBulkPs = (SQLServerPreparedStatement) multiDBTransactionManager.prepareStatement(insertSQL)) {
insertBulkPs.setLong(1, stbTransactionLog.getGroupId());
insertBulkPs.setObject(2, stbTransactionLog.getCreatedAt());
insertBulkPs.setString(3, stbTransactionLog.getUserName());
insertBulkPs.setString(4, stbTransactionLog.getEntities());
insertBulkPs.setString(5, stbTransactionLog.getEntitiesJson());
insertBulkPs.setLong(6, stbTransactionLog.getGroupId());
insertBulkPs.executeLargeUpdate();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
}

4
ems-core/src/main/java/it/integry/ems/sync/MultiDBTransaction/MultiDBTransactionManager.java
View File

@@ -46,6 +46,10 @@ public class MultiDBTransactionManager implements AutoCloseable{
public MultiDBTransactionManager() {
}
public MultiDBTransactionManager(AvailableConnectionsModel connectionsModel) throws Exception {
this(connectionsModel.getProfileName(), true);
}
public MultiDBTransactionManager(String profileDb) throws Exception {
this(profileDb, true);
}

86
ems-core/src/main/java/it/integry/ems/sync/dto/ExportHistoryGroupDTO.java
View File

@@ -1,11 +1,27 @@
package it.integry.ems.sync.dto;
import it.integry.ems.datasource.DataSource;
import javafx.util.Pair;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
public class ExportHistoryGroupDTO {
private DataSource dataSource;
private long groupId;
private List<ExportHistoryItemDTO> items;
private LocalDateTime startTime;
private LocalDateTime endTime;
private boolean completedRead;
private long totalItemCount;
private long processedItemCount;
private long syncronizedItemCount;
private List<ExportHistoryItemDTO> items = new ArrayList<>();
private final Queue<Pair<String, String>> toProcessQueue = new ConcurrentLinkedQueue<>();
public long getGroupId() {
@@ -25,4 +41,72 @@ public class ExportHistoryGroupDTO {
this.items = items;
return this;
}
public Queue<Pair<String, String>> getToProcessQueue() {
return toProcessQueue;
}
public LocalDateTime getStartTime() {
return startTime;
}
public ExportHistoryGroupDTO setStartTime(LocalDateTime startTime) {
this.startTime = startTime;
return this;
}
public DataSource getDataSource() {
return dataSource;
}
public ExportHistoryGroupDTO setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
return this;
}
public long getTotalItemCount() {
return totalItemCount;
}
public ExportHistoryGroupDTO setTotalItemCount(long totalItemCount) {
this.totalItemCount = totalItemCount;
return this;
}
public long getProcessedItemCount() {
return processedItemCount;
}
public ExportHistoryGroupDTO setProcessedItemCount(long processedItemCount) {
this.processedItemCount = processedItemCount;
return this;
}
public long getSyncronizedItemCount() {
return syncronizedItemCount;
}
public ExportHistoryGroupDTO setSyncronizedItemCount(long syncronizedItemCount) {
this.syncronizedItemCount = syncronizedItemCount;
return this;
}
public boolean isCompletedRead() {
return completedRead;
}
public ExportHistoryGroupDTO setCompletedRead(boolean completedRead) {
this.completedRead = completedRead;
return this;
}
public LocalDateTime getEndTime() {
return endTime;
}
public ExportHistoryGroupDTO setEndTime(LocalDateTime endTime) {
this.endTime = endTime;
return this;
}
}

65
ems-core/src/main/java/it/integry/ems/sync/dto/ExportHistoryItemDTO.java
View File

@@ -1,41 +1,12 @@
package it.integry.ems.sync.dto;
import it.integry.ems.datasource.DataSource;
import it.integry.ems_model.base.EntityBase;
import java.time.LocalDateTime;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
public class ExportHistoryItemDTO {
private long publicationGroupId;
private DataSource dataSource;
private PublicationDTO publication;
private LocalDateTime startDate;
private long totalItemCount;
private long processedItemCount;
private long syncronizedItemCount;
private Queue<EntityBase> toProcessQueue = new ConcurrentLinkedQueue<EntityBase>();
public long getPublicationGroupId() {
return publicationGroupId;
}
public ExportHistoryItemDTO setPublicationGroupId(long publicationGroupId) {
this.publicationGroupId = publicationGroupId;
return this;
}
public DataSource getDataSource() {
return dataSource;
}
public ExportHistoryItemDTO setDataSource(DataSource dataSource) {
this.dataSource = dataSource;
return this;
}
public PublicationDTO getPublication() {
return publication;
@@ -54,40 +25,4 @@ public class ExportHistoryItemDTO {
this.startDate = startDate;
return this;
}
public long getTotalItemCount() {
return totalItemCount;
}
public ExportHistoryItemDTO setTotalItemCount(long totalItemCount) {
this.totalItemCount = totalItemCount;
return this;
}
public long getProcessedItemCount() {
return processedItemCount;
}
public ExportHistoryItemDTO setProcessedItemCount(long processedItemCount) {
this.processedItemCount = processedItemCount;
return this;
}
public Queue<EntityBase> getToProcessQueue() {
return toProcessQueue;
}
public ExportHistoryItemDTO setToProcessQueue(Queue<EntityBase> toProcessQueue) {
this.toProcessQueue = toProcessQueue;
return this;
}
public long getSyncronizedItemCount() {
return syncronizedItemCount;
}
public ExportHistoryItemDTO setSyncronizedItemCount(long syncronizedItemCount) {
this.syncronizedItemCount = syncronizedItemCount;
return this;
}
}

12
ems-core/src/main/java/it/integry/ems/sync/service/RemoteSynchronizationService.java
View File

@@ -5,11 +5,10 @@ import it.integry.ems.sync.AsyncHistoryManager;
import it.integry.ems.sync.MultiDBTransaction.MultiDBTransactionManager;
import it.integry.ems.sync.dto.ExportHistoryStatusDTO;
import it.integry.ems.sync.dto.PublicationDTO;
import it.integry.ems.utility.UtilityEntity;
import it.integry.ems_model.base.EntityBase;
import it.integry.ems_model.base.EntityPropertyHolder;
import it.integry.ems_model.entity.StbPublicationsDetail;
import it.integry.ems_model.utility.Query;
import it.integry.ems_model.utility.UtilityDB;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.beans.factory.annotation.Autowired;
@@ -37,11 +36,12 @@ public class RemoteSynchronizationService {
private EntityProcessor entityProcessor;
public void startPublication(long groupId) throws Exception {
StbPublicationsDetail tmpStbPublicationsDetail = new StbPublicationsDetail();
String whereCond = Query.format("stb_publication_id = %s AND export_history = %s", groupId, 1);
String sql = Query.format("SELECT * FROM " + StbPublicationsDetail.ENTITY + " WHERE stb_publication_id = %s AND export_history = %s AND ready_to_transmit = 0", groupId, 1);
final List<? extends EntityBase> select = tmpStbPublicationsDetail.select(multiDBTransactionManager.getPrimaryConnection(), whereCond);
final List<StbPublicationsDetail> stbPublicationsDetails = UtilityEntity.toCustomEntity(select);
final List<StbPublicationsDetail> stbPublicationsDetails = UtilityDB.executeSimpleQueryDTO(multiDBTransactionManager.getPrimaryConnection(), sql, StbPublicationsDetail.class);
if(stbPublicationsDetails == null)
throw new Exception("Non è stata trovata alcuna pubblicazione da poter avviare (ID: " + groupId + ")");
for (StbPublicationsDetail stbPublicationsDetail : stbPublicationsDetails) {
asyncHistoryManager.addToExportQueue(multiDBTransactionManager.getPrimaryDatasource(),

6
ems-core/src/main/java/it/integry/ems/sync/service/RemoteSynchronizationSetupService.java
View File

@@ -79,7 +79,7 @@ public class RemoteSynchronizationSetupService {
.collect(toList());
publicationGroupDTO.setPublications(publications);
boolean atLeastOneActiveToExport = publications.stream().anyMatch(publicationDTO -> publicationDTO.getExportHistory() && publicationDTO.getActive());
boolean atLeastOneActiveToExport = publications.stream().anyMatch(publicationDTO -> publicationDTO.getExportHistory() && publicationDTO.getActive() && !publicationDTO.getReadyToTransmit());
boolean alreadyExporting = currentExportStatus.stream().anyMatch(x -> x.getPublicationGroupId() == publicationGroupDTO.getId());
publicationGroupDTO.setCanStartExport(atLeastOneActiveToExport && !alreadyExporting);
@@ -113,7 +113,7 @@ public class RemoteSynchronizationSetupService {
.setActive(publicationToInsert.getActive())
.setWhereCondField(publicationToInsert.getWhereCond())
.setExportHistory(publicationToInsert.getExportHistory())
.setReadyToTransmit(!publicationToInsert.getReadyToTransmit());
.setReadyToTransmit(!publicationToInsert.getExportHistory());
stbPublicationsDetail.setOperation(OperationType.INSERT);
StbPublications stbPublications = new StbPublications()
@@ -190,7 +190,7 @@ public class RemoteSynchronizationSetupService {
stbPublicationsDetail = entityProcessor.processEntity(stbPublicationsDetail, multiDBTransactionManager);
if(publicationDTO.getExportHistory() != null)
if(publicationDTO.getExportHistory() != null && !stbPublicationsDetail.isReadyToTransmit())
stbPublicationsDetail.setExportHistory(publicationDTO.getExportHistory());
if(publicationDTO.getActive() != null)

12
ems-core/src/main/java/it/integry/ems_model/base/EntityBase.java
View File

@@ -113,7 +113,7 @@ public abstract class EntityBase implements Serializable, Cloneable, EntityInter
//ID del gruppo di transazioni per le Sync OFFLINE
@JsonIgnore
private Integer transactionGroupId = null;
private Long transactionGroupId = null;
@JsonIgnore
private int queryTimeoutSeconds = 60 * 30;
@@ -927,8 +927,7 @@ public abstract class EntityBase implements Serializable, Cloneable, EntityInter
long publicationId = AsyncManager.getPublicationIdIfExists(dbName, this);
if (publicationId > 0) {
if (transactionGroupId == null) transactionGroupId = AsyncManager.getNextTransactionGroupId(connection);
AsyncManager.saveNewTransaction(connection, this, transactionGroupId);
transactionGroupId = AsyncManager.saveNewTransaction(connection, dbName, this, publicationId, transactionGroupId);
}
}
}
@@ -1988,11 +1987,11 @@ public abstract class EntityBase implements Serializable, Cloneable, EntityInter
return builder.toString();
}
public Integer getTransactionGroupId() {
public Long getTransactionGroupId() {
return transactionGroupId;
}
public void setTransactionGroupId(Integer transactionGroupId) {
public void setTransactionGroupId(Long transactionGroupId) {
this.transactionGroupId = transactionGroupId;
}
@@ -2005,8 +2004,7 @@ public abstract class EntityBase implements Serializable, Cloneable, EntityInter
try {
field.setAccessible(true);
idRiga = (Integer) field.get(entityBase);
} catch (Exception e) {
} catch (Exception ignored) {
}
return idRiga;
}

4
ems-core/src/main/java/it/integry/ems_model/base/EntityInterface.java
View File

@@ -83,9 +83,9 @@ public interface EntityInterface {
EntityException getException();
Integer getTransactionGroupId();
Long getTransactionGroupId();
void setTransactionGroupId(Integer transactionGroupId);
void setTransactionGroupId(Long transactionGroupId);
void resetTransactionGroupId();

282
ems-core/src/main/java/it/integry/ems_model/db/AsyncResultSetMapper.java Normal file
View File

@@ -0,0 +1,282 @@
package it.integry.ems_model.db;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.microsoft.sqlserver.jdbc.SQLServerResultSet;
import it.integry.ems.expansion.RunnableArgs;
import it.integry.ems.expansion.RunnableArgsWithReturn;
import it.integry.ems.utility.UtilityDebug;
import it.integry.ems_model.ImportExport.Interfaces.IExportFormat;
import it.integry.ems_model.ImportExport.Interfaces.IImportFormat;
import it.integry.ems_model.annotation.Clob;
import it.integry.ems_model.annotation.MapToTable;
import it.integry.ems_model.annotation.SqlField;
import it.integry.ems_model.base.EntityBase;
import it.integry.ems_model.exception.DataConverterNotFoundException;
import it.integry.ems_model.resolver.SqlFieldHolder;
import it.integry.ems_model.types.OperationType;
import it.integry.ems_model.utility.UtilityDB;
import it.integry.ems_model.utility.UtilityString;
import org.apache.commons.codec.binary.Base64;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import javax.validation.constraints.NotNull;
import java.lang.reflect.Field;
import java.lang.reflect.Type;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.*;
public class AsyncResultSetMapper {
private final Logger logger = LogManager.getLogger();
@NotNull
public static List<HashMap<String, Object>> mapResultSetToHashMap(ResultSet rs) throws SQLException {
ResultSetMetaData md = rs.getMetaData();
int columns = md.getColumnCount();
List<HashMap<String, Object>> list = new ArrayList<HashMap<String, Object>>();
while (rs.next()) {
HashMap<String, Object> row = new HashMap<String, Object>(columns);
for (int i = 1; i <= columns; ++i) {
row.put(md.getColumnName(i), rs.getObject(i));
}
list.add(row);
}
return list;
}
public static Map.Entry<String[], List<Object[]>> mapResultSetToArray(ResultSet rs) throws SQLException {
ResultSetMetaData md = rs.getMetaData();
int columns = md.getColumnCount();
String[] columnNames = new String[columns];
for (int i = 1; i <= columns; ++i) {
columnNames[i - 1] = md.getColumnName(i).toLowerCase();
}
List<Object[]> rowsValues = new ArrayList<>();
while (rs.next()) {
Object[] singleRowValues = new Object[columns];
for (int i = 1; i <= columns; ++i) {
singleRowValues[i - 1] = rs.getObject(i);
}
rowsValues.add(singleRowValues);
}
rs.close();
return new AbstractMap.SimpleEntry<>(columnNames, rowsValues);
}
public <T> void mapResultSetToList(SQLServerResultSet rs, Class<T> outputClass, RunnableArgs<T> onElementRetrieved) throws SQLException, DataConverterNotFoundException, InstantiationException, IllegalAccessException {
mapResultSetToList(rs, outputClass, null, true, onElementRetrieved);
}
public <T> void mapResultSetToList(SQLServerResultSet rs, Class<T> outputClass, OperationType operation, RunnableArgs<T> onElementRetrieved)
throws SQLException, DataConverterNotFoundException, InstantiationException, IllegalAccessException {
mapResultSetToList(rs, outputClass, operation, true, onElementRetrieved);
}
public <T> void mapResultSetToList(SQLServerResultSet rs, Class<T> outputClass, OperationType operation, Boolean onlyPKMaster, RunnableArgs<T> onElementRetrieved) throws SQLException, DataConverterNotFoundException, InstantiationException, IllegalAccessException {
if (rs != null) {
Field[] fields;
if (IExportFormat.class.isAssignableFrom(outputClass) || IImportFormat.class.isAssignableFrom(outputClass)) {
fields = outputClass.getSuperclass().getDeclaredFields();
} else {
fields = outputClass.getDeclaredFields();
}
internalMappingList(rs, outputClass, fields, onElementRetrieved);
// if (outputList != null && !outputList.isEmpty() && EntityBase.class.isAssignableFrom(outputClass)) {
// for (T bean : outputList) {
// if (operation != null) {
// ((EntityBase) bean).setOperation(operation);
// }
//
// if (onlyPKMaster != null) {
// ((EntityBase) bean).setOnlyPkMaster(onlyPKMaster);
// }
// }
// }
}
// return outputList != null && !outputList.isEmpty() ? outputList : null;
}
private <T> void internalMappingList(SQLServerResultSet resultSet, Class<T> outputClass, Field[] fields, RunnableArgs<T> onElementRetrieved) throws SQLException, DataConverterNotFoundException, InstantiationException, IllegalAccessException {
final HashMap<Integer, RunnableArgsWithReturn<Object, Object>> columnToConverterMap = new HashMap<>();
final HashMap<Integer, Field> columnToFieldMap = new HashMap<>();
final ResultSetMetaData metaData = resultSet.getMetaData();
for (int i = 0; i < metaData.getColumnCount(); i++) {
String columnName = metaData.getColumnName(i + 1).toLowerCase();
final Field field = Arrays.stream(fields)
.filter(x -> x.isAnnotationPresent(SqlField.class) && x.getAnnotation(SqlField.class).value().equalsIgnoreCase(columnName))
.findFirst().orElse(null);
if (field == null)
continue;
field.setAccessible(true);
final int columnType = metaData.getColumnType(i + 1);
final RunnableArgsWithReturn<Object, Object> converter = SqlFieldHolder.getConverter(UtilityDB.sqlTypeToJavaClass(columnType), field.getType());
columnToConverterMap.put(i + 1, converter);
columnToFieldMap.put(i + 1, field);
}
while (resultSet.next()) {
final T outputObject = outputClass.newInstance();
for (Integer columnIndex : columnToConverterMap.keySet()) {
Object obj = resultSet.getObject(columnIndex);
Object convertedObject = columnToConverterMap.get(columnIndex).run(obj);
columnToFieldMap.get(columnIndex).set(outputObject, convertedObject);
}
onElementRetrieved.run(outputObject);
}
//
// final Map.Entry<String[], List<Object[]>> listPair = mapResultSetToArray(resultSet);
//
// List<String> columnNames = Arrays.asList(listPair.getKey());
// List<Object[]> columnValuesList = listPair.getValue();
//
// if (columnValuesList == null || columnValuesList.isEmpty())
// return null;
//
// List<Field> fieldsWithAnnotations = Stream.of(fields)
// .filter(x -> x.isAnnotationPresent(SqlField.class) || x.isAnnotationPresent(JsonProperty.class) || x.isAnnotationPresent(MapToTable.class))
// .toList();
//
// if (fieldsWithAnnotations.isEmpty()) {
// System.out.println("Attenzione nella classe non sono presenti elementi con le seguenti annotazioni @SqlField, @JsonProperty, @MapToTable. Nessun campo sarà mappato nella classe");
// }
//
//
// final List<ResultSetMappingField> resultSetMappingFields = scanResultSetFields(fields, columnNames, columnTypesFromResultSet);
// final HashMap<Field, List<ResultSetMappingField>> entityChildsResultSetMappingField = new HashMap<>();
//
// //Searching entities
// List<Field> entityFields = fieldsWithAnnotations.stream()
// .filter(x -> EntityBase.class.isAssignableFrom(x.getType()))
// .collect(Collectors.toList());
//
// for (Field entityField : entityFields) {
// Field[] entityChildFields = entityField.getType().getDeclaredFields();
// entityChildsResultSetMappingField.put(entityField, scanResultSetFields(entityChildFields, columnNames, columnTypesFromResultSet));
// }
//
//
//
// for (int i = 0; i < listPair.getValue().size(); i++) {
// T bean = outputClass.newInstance();
// Object[] columnValues = columnValuesList.get(i);
//
// for (ResultSetMappingField resultSetMappingField : resultSetMappingFields) {
// Object columnValue = columnValues[resultSetMappingField.getColumnIndex()];
//
// if (resultSetMappingField.getConverter() != null && columnValue != null) {
// columnValue = resultSetMappingField.getConverter().run(columnValue);
// resultSetMappingField.getField().set(bean, columnValue);
// }
// }
//
// for (Field entityChildResultSetMappingField : entityChildsResultSetMappingField.keySet()) {
// EntityBase entityChildBean = (EntityBase) entityChildResultSetMappingField.getType().newInstance();
// entityChildBean.setOperation(OperationType.SELECT_OBJECT);
// entityChildResultSetMappingField.set(bean, entityChildBean);
//
// List<ResultSetMappingField> entityChildResultSetMappingFieldList = entityChildsResultSetMappingField.get(entityChildResultSetMappingField);
//
// for (ResultSetMappingField entityChildResultSetMapping : entityChildResultSetMappingFieldList) {
// Object columnValue = columnValues[entityChildResultSetMapping.getColumnIndex()];
//
// if (entityChildResultSetMapping.getConverter() != null && columnValue != null) {
// columnValue = entityChildResultSetMapping.getConverter().run(columnValue);
// entityChildResultSetMapping.getField().set(entityChildBean, columnValue);
// }
// }
//
// }
//
// values.add(bean);
// }
//
// return values;
}
private List<ResultSetMappingField> scanResultSetFields(Field[] fields, List<String> columnNames, HashMap<String, Type> sqlTypes) throws DataConverterNotFoundException {
final List<ResultSetMappingField> resultSetMappingFields = new ArrayList<>();
for (Field field : fields) {
field.setAccessible(true);
if (!field.isAnnotationPresent(SqlField.class) && !field.isAnnotationPresent(JsonProperty.class) && !field.isAnnotationPresent(MapToTable.class)) {
continue;
}
String columnName;
if (field.isAnnotationPresent(SqlField.class)) {
SqlField column = field.getAnnotation(SqlField.class);
columnName = !UtilityString.isNullOrEmpty(column.value()) ? column.value() : field.getName();
} else if (field.isAnnotationPresent(MapToTable.class)) {
columnName = field.getAnnotation(MapToTable.class).value();
} else if (field.isAnnotationPresent(JsonProperty.class)) {
columnName = field.getAnnotation(JsonProperty.class).value();
} else continue;
if (EntityBase.class.isAssignableFrom(field.getType()))
continue;
columnName = columnName.toLowerCase();
Type sqlType = sqlTypes.get(columnName);
Type entityType = field.getGenericType();
if (sqlType == null)
continue;
RunnableArgsWithReturn<Object, Object> converter;
if (field.isAnnotationPresent(Clob.class)) {
converter = data -> Base64.encodeBase64String(data.toString().getBytes());
} else
converter = SqlFieldHolder.getConverter(sqlType, entityType);
if (converter == null) {
if (UtilityDebug.isDebugExecution() || UtilityDebug.isIntegryServer())
throw new DataConverterNotFoundException(columnName, sqlType.getTypeName(), entityType.getTypeName());
logger.warn(String.format("Converter non trovato per colonna %s, sqlType %s -> dtoType %s", columnName, sqlType.getTypeName(), entityType.getTypeName()));
}
resultSetMappingFields.add(new ResultSetMappingField()
.setField(field)
.setColumnName(columnName)
.setFieldName(field.getName())
.setConverter(converter)
.setColumnIndex(columnNames.indexOf(columnName)));
}
return resultSetMappingFields;
}
}

11
ems-core/src/main/java/it/integry/ems_model/entity/DtbDoct.java
View File

@@ -316,6 +316,9 @@ public class DtbDoct extends DtbBaseDocT {
@SqlField(value = "cod_lingua", nullable = true)
private String codLingua;
@SqlField(value = "id_lotto", nullable = true)
private Integer idLotto;
@JsonProperty(value = "explodeKit")
private String explodeKit;
@@ -1206,6 +1209,14 @@ public class DtbDoct extends DtbBaseDocT {
return this;
}
public Integer getIdLotto() {
return idLotto;
}
public void setIdLotto(Integer idLotto) {
this.idLotto = idLotto;
}
@Override
public String getExplodeKit() {
return explodeKit;

18
ems-core/src/main/java/it/integry/ems_model/entity/StbPublicationsDetail.java
View File

@@ -27,13 +27,13 @@ public class StbPublicationsDetail extends EntityBase {
private String entityName;
@SqlField(value = "active", defaultObjectValue="1", nullable = false)
private boolean active;
private Boolean active;
@SqlField(value = "export_history", defaultObjectValue="0", nullable = false)
private boolean exportHistory;
private Boolean exportHistory;
@SqlField(value = "ready_to_transmit", defaultObjectValue="0", nullable = false)
private boolean readyToTransmit;
private Boolean readyToTransmit;
@SqlField(value = "recalc_columns")
private String recalcColumnsField;
@@ -70,29 +70,29 @@ public class StbPublicationsDetail extends EntityBase {
return this;
}
public boolean isActive() {
public Boolean isActive() {
return active;
}
public StbPublicationsDetail setActive(boolean active) {
public StbPublicationsDetail setActive(Boolean active) {
this.active = active;
return this;
}
public boolean isExportHistory() {
public Boolean isExportHistory() {
return exportHistory;
}
public StbPublicationsDetail setExportHistory(boolean exportHistory) {
public StbPublicationsDetail setExportHistory(Boolean exportHistory) {
this.exportHistory = exportHistory;
return this;
}
public boolean isReadyToTransmit() {
public Boolean isReadyToTransmit() {
return readyToTransmit;
}
public StbPublicationsDetail setReadyToTransmit(boolean readyToTransmit) {
public StbPublicationsDetail setReadyToTransmit(Boolean readyToTransmit) {
this.readyToTransmit = readyToTransmit;
return this;
}

6
ems-core/src/main/java/it/integry/ems_model/entity/StbTransactionLog.java
View File

@@ -39,7 +39,7 @@ public class StbTransactionLog extends EntityBase {
private String entitiesJson;
@SqlField(value = "group_id", nullable = false)
private int groupId;
private long groupId;
public long getId() {
return id;
@@ -95,11 +95,11 @@ public class StbTransactionLog extends EntityBase {
return this;
}
public int getGroupId() {
public long getGroupId() {
return groupId;
}
public StbTransactionLog setGroupId(int groupId) {
public StbTransactionLog setGroupId(long groupId) {
this.groupId = groupId;
return this;
}

15
ems-core/src/main/java/it/integry/ems_model/utility/UtilityDB.java
View File

@@ -1,11 +1,15 @@
package it.integry.ems_model.utility;
import com.annimon.stream.Stream;
import com.microsoft.sqlserver.jdbc.SQLServerResultSet;
import com.microsoft.sqlserver.jdbc.SQLServerStatement;
import it.integry.common.var.CommonConstants;
import it.integry.ems.expansion.RunnableArgs;
import it.integry.ems_model.annotation.EntityChild;
import it.integry.ems_model.base.EntityBase;
import it.integry.ems_model.base.EntityPropertyHolder;
import it.integry.ems_model.config.EmsRestConstants;
import it.integry.ems_model.db.AsyncResultSetMapper;
import it.integry.ems_model.db.ResultSetMapper;
import it.integry.ems_model.exception.DataConverterNotFoundException;
import it.integry.ems_model.types.OperationType;
@@ -529,6 +533,17 @@ public class UtilityDB {
return executeSimpleQueryDTO(conn, querySql, clazz, null);
}
public static <T> void executeSimpleQueryDTOAsync(Connection conn, String querySql, Class<T> clazz, RunnableArgs<T> onElementRetrieved) throws SQLException, DataConverterNotFoundException, InstantiationException, IllegalAccessException {
try (SQLServerStatement ps = (SQLServerStatement) conn.createStatement()) {
ps.setResponseBuffering("adaptive");
try (SQLServerResultSet rs = (SQLServerResultSet) ps.executeQuery(querySql)) {
new AsyncResultSetMapper()
.mapResultSetToList(rs, clazz, OperationType.SELECT_OBJECT, onElementRetrieved);
}
}
}
@Nullable
public static <T> List<T> executeSimpleQueryDTO(Connection conn, String querySql, Class<T> clazz, Integer timeout) throws SQLException, DataConverterNotFoundException, InstantiationException, IllegalAccessException {

1
ems-engine/src/main/java/it/integry/ems/retail/ReportVariazioni/dto/Variazioni/VariazioniPvInputDTO.java
View File

@@ -103,6 +103,7 @@ public class VariazioniPvInputDTO {
public enum Type {
VARIAZIONI("V"),
PROMOZIONI("P"),
GRIGLIA("G"),
LISTINO("L");
private String text;

43
ems-engine/src/main/java/it/integry/ems/retail/ReportVariazioni/service/VariazioniPVService.java
View File

@@ -123,7 +123,14 @@ public class VariazioniPVService {
String query = getQueryVariazioni(variazioniPvDTO, setup, codVlis, getSetupSectionDepo.get("SELEZIONA_GRIGLIE"));
List<HashMap<String, Object>> elencoVar = UtilityDB.executeSimpleQuery(multiDBTransactionManager.getPrimaryConnection(), query);
if (getSetupSectionDepo.get("SELEZIONA_GRIGLIE") != null && getSetupSectionDepo.get("SELEZIONA_GRIGLIE").equalsIgnoreCase("S")) {
boolean getVariazioneGriglia = false;
if ((variazioniPvDTO.getTipoReportEnum() == VariazioniPvInputDTO.Type.GRIGLIA ||
variazioniPvDTO.getTipoReportEnum() == VariazioniPvInputDTO.Type.LISTINO) &&
getSetupSectionDepo.get("SELEZIONA_GRIGLIE") != null &&
getSetupSectionDepo.get("SELEZIONA_GRIGLIE").equalsIgnoreCase("S")) {
getVariazioneGriglia = true;
grigliaAcquisto = grigliaAcquistoHandlerService.getGrigliaAcquisto();
grigliaAcquisto = Stream.of(grigliaAcquisto)
.filter(x -> x.getDataValidita().equals(variazioniPvDTO.getDataValidita()) &&
@@ -135,7 +142,7 @@ public class VariazioniPVService {
CodMartType codMart = fillArticoloData(objectFactory,
variazioni, variazioniPvDTO, invioIngredienti, getSetupSectionDepo, codMdep, colNameDescrEstesa,
setup.get("LIBERO1"),
setup.get("LIBERO2"));
setup.get("LIBERO2"), getVariazioneGriglia);
listinoDTO.getContent().add(objectFactory.createCodVlisTypeCodMart(codMart));
}
}
@@ -156,7 +163,8 @@ public class VariazioniPVService {
String codMdep,
String colNameDescrEstesa,
String valLibero1,
String valLibero2) throws Exception {
String valLibero2,
boolean getVariazioneGriglia) throws Exception {
CodMartType codMartType = objectFactory.createCodMartType();
String codMart = UtilityString.streNull(UtilityHashMap.getValueIfExists(var, "cod_mart"));
String flagTipoPromo = UtilityString.streNull(UtilityHashMap.getValueIfExists(var, "flag_tipo_promo"));
@@ -325,7 +333,7 @@ public class VariazioniPVService {
//DATI GRIGLIA DI ACQUISTO
if ("S".equalsIgnoreCase(setupDepo.get("SELEZIONA_GRIGLIE"))) {
getDatiGriglia(objectFactory, codMart, codMdep, variazioniPvDTO.getDataValidita(), codMartType);
getDatiGriglia(objectFactory, codMart, codMdep, variazioniPvDTO.getDataValidita(), codMartType, getVariazioneGriglia);
}
if ("S".equalsIgnoreCase(setupDepo.get("ABILITA_CAMPI_LIBERI"))) {
@@ -612,6 +620,22 @@ public class VariazioniPVService {
.replace("'[COD_VLIS]'", UtilityDB.valueToString(codVlis))
.replace("'[DATA_VALIDITA]'", UtilityDB.valueDateToString(variazioniPvDTO.getDataValidita(), CommonConstants.DATE_FORMAT_YMD));
break;
case GRIGLIA:
queryAssortimento =
"SELECT mtb_depo.cod_vlis,\n" +
" atb_griglie_art.cod_mart,\n" +
" 'G' AS tipo_variazione,\n" +
" CONVERT(VARCHAR(15), NULL) AS cod_mart_view,\n" +
" 'N' AS 'fine_promo'\n" +
"FROM atb_griglie_art\n" +
" INNER JOIN mtb_depo ON atb_griglie_art.cod_mdep = mtb_depo.cod_mdep\n" +
"WHERE atb_griglie_art.data_validita BETWEEN DateAdd(day, -1, '[DATA_VALIDITA]') AND '[DATA_VALIDITA]'";
queryAssortimento =
queryAssortimento
.replace("'[COD_VLIS]'", UtilityDB.valueToString(codVlis))
.replace("'[DATA_VALIDITA]'", UtilityDB.valueDateToString(variazioniPvDTO.getDataValidita(), CommonConstants.DATE_FORMAT_YMD));
break;
case PROMOZIONI:
String filtroPromoOrto = setup.get("FILTRO_ART_ORTOFRUTTA");
queryAssortimento =
@@ -853,9 +877,15 @@ public class VariazioniPVService {
return codiciABarreType;
}
private void getDatiGriglia(ObjectFactory objectFactory, String codMart, String codMdep, Date dataValidita, CodMartType codMartType) throws Exception {
private void getDatiGriglia(ObjectFactory objectFactory, String codMart, String codMdep, Date dataValidita, CodMartType codMartType, boolean getVariazioneGriglia) throws Exception {
CodAlisType codAlisType = objectFactory.createCodAlisType();
String tipoAssortimento = "",
tipoVariazioneGriglia = "",
codAlis = "",
descrList = "";
if (getVariazioneGriglia) {
List<GrigliaAcquistoDTO> grigliaArticolo = Stream.of(grigliaAcquisto)
.filter(x -> x.getCodMart().equalsIgnoreCase(codMart) &&
x.getCodMdep().equalsIgnoreCase(codMdep) &&
@@ -905,8 +935,6 @@ public class VariazioniPVService {
}
}
String tipoAssortimento, tipoVariazioneGriglia, codAlis, descrList = null;
if (grigliaArticolo != null && !grigliaArticolo.isEmpty()) {
codAlis = grigliaArticolo.get(0).getCodAlis();
tipoAssortimento = UtilityString.streNull(grigliaArticolo.get(0).getTipoAssortimento());
@@ -928,6 +956,7 @@ public class VariazioniPVService {
tipoAssortimento = "F.A.";
tipoVariazioneGriglia = "";
}
}
codAlisType.setValue(codAlis);
codAlisType.setDescrizione(descrList);

2
pom.xml
View File

@@ -44,7 +44,7 @@
<brt.tracknumber.listener.enabled>true</brt.tracknumber.listener.enabled>
<custom.object.mapper>jacksonResponseObjectMapper</custom.object.mapper>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<gson.version>2.5</gson.version>
<gson.version>2.9.1</gson.version>
<httpclient.version>4.3.6</httpclient.version>
</properties>