package com.megatim.fdxconvert.service;
|
|
import com.megatim.typefichier.validator.Validator;
|
import com.megatim.typefichier.validator.model.ConfigJsonValidator;
|
import com.megatim.typefichier.validator.model.ConfigStreamValidator;
|
import static com.megatim.typefichier.validator.utilities.Utilities.getCharset;
|
import com.megatim.fdxconvert.controller.FileToValidateEditDialogController;
|
import com.megatim.fdxconvert.dao.TypeFichierJsonDAO;
|
import com.megatim.fdxconvert.dao.ValidateurDAO;
|
import com.megatim.fdxconvert.enums.DataType;
|
import static com.megatim.fdxconvert.enums.DataType.CSV;
|
import static com.megatim.fdxconvert.enums.DataType.TXT;
|
import static com.megatim.fdxconvert.enums.DataType.XLS;
|
import static com.megatim.fdxconvert.enums.DataType.XLSX;
|
import com.megatim.fdxconvert.enums.JournalStatut;
|
import com.megatim.fdxconvert.model.AlphaNumeriqueField;
|
import com.megatim.fdxconvert.model.Configuration;
|
import com.megatim.fdxconvert.model.Journal;
|
import com.megatim.fdxconvert.model.MetaAlphaNumeriqueField;
|
import com.megatim.fdxconvert.model.Tache;
|
import com.megatim.fdxconvert.model.TypeFichierJson;
|
import com.megatim.fdxconvert.model.Validateur;
|
import com.megatim.fdxconvert.util.ImportData;
|
import com.megatim.fdxconvert.pojo.FileToValidateDescription;
|
import com.megatim.fdxconvert.util.TypeFichierJsonConverter;
|
import com.megatim.fdxconvert.util.Utilities;
|
import java.io.BufferedReader;
|
import java.io.File;
|
import java.io.FileInputStream;
|
import java.io.IOException;
|
import java.io.InputStreamReader;
|
import java.nio.charset.Charset;
|
import java.nio.file.Path;
|
import java.nio.file.Paths;
|
import java.time.LocalDateTime;
|
import java.time.format.DateTimeFormatter;
|
import java.util.HashSet;
|
import java.util.Optional;
|
import java.util.Random;
|
import java.util.Set;
|
import java.util.logging.Level;
|
import java.util.logging.Logger;
|
import org.apache.commons.io.IOUtils;
|
import org.quartz.Job;
|
import org.quartz.JobDataMap;
|
import org.quartz.JobExecutionContext;
|
import org.quartz.JobExecutionException;
|
|
/**
|
*
|
* @author MGT_DEV3
|
*/
|
public class TacheJob implements Job {
|
|
@Override
|
public void execute(JobExecutionContext context) throws JobExecutionException {
|
|
JobDataMap dataMap = context.getJobDetail().getJobDataMap();
|
|
Tache tache = TacheService.getInstance().findByTacheId(dataMap.getString("tacheId"));
|
|
//On peut concrètement exploitater la tâche ici !!
|
File sourceDirectory = new File(tache.getRepertoireSource());
|
|
if (sourceDirectory.exists() && sourceDirectory.isDirectory()) {
|
|
Configuration config = ConfigurationService.getInstance().getCurrentConfig();
|
|
for (File file : sourceDirectory.listFiles()) {
|
|
if (file.isFile()) {
|
|
try {
|
|
if (tache.getDataType().equals(DataType.JSON)) {
|
validateJsonFile(tache, file, config);
|
|
} else {
|
//On converti le fichier à valider en fichier
|
generateAndParseTxtFile(tache, file, config);
|
}
|
|
} catch (Exception ex) {
|
Logger.getLogger(TacheJob.class.getName()).log(Level.SEVERE, ex.getMessage(), ex);
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
private void generateAndParseTxtFile(Tache tache, File fileToValidate, Configuration config) throws Exception {
|
Charset charset = getCharset(fileToValidate);
|
long lignes = numberOfLines(fileToValidate, charset);
|
|
if (config == null) {
|
saveJournal(fileToValidate.getName(), Integer.parseInt("" + lignes), JournalStatut.ECHEC_REPERTOIRE, tache.getTypeFichier().getCode(), fileToValidate.getParent());
|
return;
|
}
|
|
Optional<Validateur> validateur = ValidateurDAO.findByCodeTypeFichier(tache.getTypeFichier().getCode());
|
|
if (!validateur.isPresent()) {
|
saveJournal(fileToValidate.getName(), Integer.parseInt("" + lignes), JournalStatut.ECHEC_VALIDATEUR, tache.getTypeFichier().getCode(), fileToValidate.getParent());
|
return;
|
}
|
|
String fileName = tache.getTypeFichier().getCode() + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) + generateTierce() + ".txt";
|
|
File outputFile = new File(config.getDatasDir(), fileName);
|
|
if (!outputFile.exists()) {
|
|
//Creation du fichier
|
outputFile.createNewFile();
|
}
|
|
FileToValidateDescription raw;
|
|
Set<MetaAlphaNumeriqueField> metaAlphaFields = tache.getMetaAlphaNumeriqueFields();
|
Set<AlphaNumeriqueField> alphaFields = new HashSet<>();
|
|
metaAlphaFields.stream().forEach(m -> {
|
alphaFields.add(m.getAlphaNumeriqueField());
|
});
|
|
switch (tache.getDataType()) {
|
|
case TXT:
|
|
raw = new FileToValidateDescription(fileToValidate, tache.getRowDeliminter(), tache.getColDeliminter());
|
|
if (tache.isStrictValidation()) {
|
|
ImportData.parseFileByFieldLength(raw, validateur.get(), outputFile, tache.isWithHeader());
|
|
} else {
|
|
ImportData.parseFileCharacterByCharacter(raw, validateur.get(), outputFile, tache.isWithHeader(), alphaFields);
|
}
|
|
break;
|
|
case CSV:
|
|
raw = new FileToValidateDescription(fileToValidate, tache.getRowDeliminter(), tache.getColDeliminter());
|
ImportData.parseCsvFile(raw, validateur.get(), outputFile, tache.isWithHeader(), alphaFields);
|
|
break;
|
|
case XLS:
|
ImportData.parseXlsFile(fileToValidate.getAbsolutePath(), tache.isWithHeader(), outputFile, validateur.get(), alphaFields);
|
break;
|
case XLSX:
|
|
ImportData.parseXlsxFile(fileToValidate.getAbsolutePath(), tache.isWithHeader(), outputFile, validateur.get(), alphaFields);
|
break;
|
|
default:
|
break;
|
}
|
saveJournal(fileToValidate.getName(), Integer.parseInt("" + lignes), JournalStatut.SUCCES_GEN_TXT, tache.getTypeFichier().getCode(), fileToValidate.getParent());
|
validate(tache, outputFile, fileToValidate, config);
|
|
}
|
|
private void validateJsonFile(Tache tache, File fileToValidate, Configuration config) throws Exception {
|
Path errorPath = Paths.get(tache.getRepertoireErreur(),
|
tache.getTypeFichier().getCode() + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyyMMddHHmmss")) + generateTierce() + "_JSON.txt");
|
Charset charset = getCharset(fileToValidate);
|
int lignes = numberOfLines(fileToValidate, charset);
|
|
if (config == null) {
|
saveJournal(fileToValidate.getName(), lignes, JournalStatut.ECHEC_REPERTOIRE, tache.getTypeFichier().getCode(), fileToValidate.getParent());
|
return;
|
}
|
TypeFichierJson typefichierJson = TypeFichierJsonDAO.getByTypeFichier(tache.getTypeFichier().getCode());
|
|
if (typefichierJson == null) {
|
saveJournal(fileToValidate.getName(), lignes, JournalStatut.ECHEC_MODELE_JSON, tache.getTypeFichier().getCode(), fileToValidate.getParent());
|
return;
|
}
|
Validator validator = new Validator();
|
ConfigJsonValidator configJson = new ConfigJsonValidator(
|
errorPath.toString(),
|
tache.getRepertoireDestination(),
|
fileToValidate,
|
new TypeFichierJsonConverter(typefichierJson).convert());
|
|
boolean result = validator.validate(configJson, "", true, true, Utilities.getNbThreads());
|
|
if (result) {
|
saveJournal(fileToValidate.getName(), lignes, JournalStatut.SUCCES, tache.getTypeFichier().getCode(), tache.getRepertoireDestination());
|
|
} else {
|
saveJournal(fileToValidate.getName(), lignes, JournalStatut.ECHEC, tache.getTypeFichier().getCode(), tache.getRepertoireErreur());
|
}
|
|
}
|
|
private void saveJournal(String fileName, int lignes, JournalStatut statut, String codeTypeFichier, String sourceDirectory) throws Exception {
|
Journal journal = new Journal(LocalDateTime.now(), statut, codeTypeFichier);
|
journal.setSourceDirectory(sourceDirectory);
|
journal.setSourceFileName(fileName);
|
journal.setNombreLignes(lignes);
|
|
JournalService.getInstance().add(journal);
|
}
|
|
/**
|
*
|
* @param tache
|
* @param fileToValidate : Fichier contenant les données normalisées
|
* @param originalFile : Fichier d'origine contenant les données brutes
|
* @param config
|
* @throws Exception
|
*/
|
private void validate(Tache tache, File outputFile, File originalFile, Configuration config) throws Exception {
|
|
Charset charset = getCharset(outputFile);
|
final int lignes = numberOfLines(outputFile, charset);
|
|
if (config == null) {
|
saveJournal(outputFile.getName(), lignes, JournalStatut.ECHEC_REPERTOIRE, tache.getTypeFichier().getCode(), outputFile.getParent());
|
return;
|
}
|
|
Optional<Validateur> validateur = ValidateurDAO.findByCodeTypeFichier(tache.getTypeFichier().getCode());
|
|
if (!validateur.isPresent()) {
|
saveJournal(outputFile.getName(), lignes, JournalStatut.ECHEC_VALIDATEUR, tache.getTypeFichier().getCode(), outputFile.getParent());
|
return;
|
}
|
|
byte[] targetArray = IOUtils.toByteArray(FileToValidateEditDialogController.class.getClassLoader().getResourceAsStream("predicatelogic-engine.xml"));
|
|
ConfigStreamValidator configValidator
|
= new ConfigStreamValidator(targetArray, validateur.get().getContent(), tache.getRepertoireErreur(),
|
tache.getRepertoireDestination(), outputFile, originalFile);
|
|
Validator validator = new Validator();
|
|
boolean result = validator.validate(configValidator, Boolean.TRUE, Boolean.TRUE, Utilities.getNbThreads());
|
|
if (result) {
|
saveJournal(outputFile.getName(), lignes, JournalStatut.SUCCES, tache.getTypeFichier().getCode(), tache.getRepertoireDestination());
|
|
} else {
|
saveJournal(outputFile.getName(), lignes, JournalStatut.ECHEC, tache.getTypeFichier().getCode(), tache.getRepertoireErreur());
|
}
|
|
}
|
|
private String generateTierce() {
|
|
// Instance of random class
|
Random rand = new Random();
|
int upperbound = 60;
|
int random = rand.nextInt(upperbound);
|
|
return String.format("%02d", random);
|
|
}
|
|
private int numberOfLines(File file, Charset charset) {
|
int nb = 0;
|
|
try ( FileInputStream fis = new FileInputStream(file); InputStreamReader isr = new InputStreamReader(fis, charset); BufferedReader reader = new BufferedReader(isr)) {
|
|
while (reader.readLine() != null) {
|
nb++;
|
}
|
} catch (IOException ex) {
|
Logger.getLogger(TacheJob.class.getName()).log(Level.SEVERE, ex.getMessage(), ex);
|
}
|
return nb;
|
}
|
}
|
