diff --git a/api/api-ingest/ingest-commons/src/main/java/fr/gouv/vitamui/ingest/common/rest/RestApi.java b/api/api-ingest/ingest-commons/src/main/java/fr/gouv/vitamui/ingest/common/rest/RestApi.java
index d84a9c1d60aaef761044e1d1f0dc4dd65c89a3f2..1671e8afc4d4a9448f83308d8cb4f0de1c3f2f07 100644
--- a/api/api-ingest/ingest-commons/src/main/java/fr/gouv/vitamui/ingest/common/rest/RestApi.java
+++ b/api/api-ingest/ingest-commons/src/main/java/fr/gouv/vitamui/ingest/common/rest/RestApi.java
@@ -29,6 +29,7 @@ package fr.gouv.vitamui.ingest.common.rest;
public class RestApi {
public static final String V1_INGEST = "/iam/v1/ingest";
public static final String INGEST_REPORT_ODT = "/odtreport";
+ public static final String INGEST_UPLOAD_V2 = "/upload-v2";
public static final String INGEST_ATR = "/atr";
public static final String INGEST_MANIFEST = "/manifest";
}
diff --git a/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClient.java b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClient.java
index bc5cfb76224f445a59755cc51e1d30feb4a86281..0ed2d094ad2557de2b5a3cb23f3a77b80e55eaab 100644
--- a/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClient.java
+++ b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClient.java
@@ -57,7 +57,8 @@ import java.util.List;
/**
* A REST client to get logbooks for an External API.
*/
-public class IngestExternalRestClient extends BasePaginatingAndSortingRestClient<LogbookOperationDto, ExternalHttpContext> {
+public class IngestExternalRestClient
+ extends BasePaginatingAndSortingRestClient<LogbookOperationDto, ExternalHttpContext> {
private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestExternalRestClient.class);
@@ -70,22 +71,27 @@ public class IngestExternalRestClient extends BasePaginatingAndSortingRestClient
return RestApi.V1_INGEST;
}
- @Override protected Class<LogbookOperationDto> getDtoClass() {
+ @Override
+ protected Class<LogbookOperationDto> getDtoClass() {
return LogbookOperationDto.class;
}
- @Override protected ParameterizedTypeReference<List<LogbookOperationDto>> getDtoListClass() {
- return new ParameterizedTypeReference<List<LogbookOperationDto>>() {};
+ @Override
+ protected ParameterizedTypeReference<List<LogbookOperationDto>> getDtoListClass() {
+ return new ParameterizedTypeReference<List<LogbookOperationDto>>() {
+ };
}
- @Override protected ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>> getDtoPaginatedClass() {
- return new ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>>() {};
+ @Override
+ protected ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>> getDtoPaginatedClass() {
+ return new ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>>() {
+ };
}
public ResponseEntity<byte[]> generateODTReport(ExternalHttpContext context, String id) {
- final UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_REPORT_ODT + CommonConstants.PATH_ID );
+ final UriComponentsBuilder uriBuilder =
+ UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_REPORT_ODT + CommonConstants.PATH_ID);
final HttpEntity<AuditOptions> request = new HttpEntity<>(buildHeaders(context));
return restTemplate.exchange(uriBuilder.build(id), HttpMethod.GET, request, byte[].class);
-
}
}
diff --git a/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClientFactory.java b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClientFactory.java
index 487bea9139bfa763e50e97929caa7b0f81bbfad3..f2a378341a390da043d63361c22a3d63ad41747b 100644
--- a/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClientFactory.java
+++ b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestExternalRestClientFactory.java
@@ -39,6 +39,7 @@ package fr.gouv.vitamui.ingest.external.client;
import fr.gouv.vitamui.commons.rest.client.BaseRestClientFactory;
import fr.gouv.vitamui.commons.rest.client.configuration.HttpPoolConfiguration;
import fr.gouv.vitamui.commons.rest.client.configuration.RestClientConfiguration;
+import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.web.client.RestTemplateBuilder;
/**
diff --git a/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClient.java b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..fdb1f227aa9ec623eb308ccc34a0d33b99f9cf17
--- /dev/null
+++ b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClient.java
@@ -0,0 +1,124 @@
+/**
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2019-2020)
+ * and the signatories of the "VITAM - Accord du Contributeur" agreement.
+ *
+ * contact@programmevitam.fr
+ *
+ * This software is a computer program whose purpose is to implement
+ * implement a digital archiving front-office system for the secure and
+ * efficient high volumetry VITAM solution.
+ *
+ * This software is governed by the CeCILL-C license under French law and
+ * abiding by the rules of distribution of free software. You can use,
+ * modify and/ or redistribute the software under the terms of the CeCILL-C
+ * license as circulated by CEA, CNRS and INRIA at the following URL
+ * "http://www.cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy,
+ * modify and redistribute granted by the license, users are provided only
+ * with a limited warranty and the software's author, the holder of the
+ * economic rights, and the successive licensors have only limited
+ * liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated
+ * with loading, using, modifying and/or developing or reproducing the
+ * software by the user in light of its specific status of free software,
+ * that may mean that it is complicated to manipulate, and that also
+ * therefore means that it is reserved for developers and experienced
+ * professionals having in-depth computer knowledge. Users are therefore
+ * encouraged to load and test the software's suitability as regards their
+ * requirements in conditions enabling the security of their systems and/or
+ * data to be ensured and, more generally, to use and operate it in the
+ * same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had
+ * knowledge of the CeCILL-C license and that you accept its terms.
+ */
+package fr.gouv.vitamui.ingest.external.client;
+
+import fr.gouv.vitam.common.model.AuditOptions;
+import fr.gouv.vitamui.commons.api.CommonConstants;
+import fr.gouv.vitamui.commons.api.domain.PaginatedValuesDto;
+import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
+import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
+import fr.gouv.vitamui.commons.rest.client.BasePaginatingAndSortingRestClient;
+import fr.gouv.vitamui.commons.rest.client.ExternalHttpContext;
+import fr.gouv.vitamui.commons.vitam.api.dto.LogbookOperationDto;
+import fr.gouv.vitamui.ingest.common.rest.RestApi;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.io.InputStream;
+import java.util.List;
+
+/**
+ * A REST client to get logbooks for an External API.
+ */
+public class IngestStreamingExternalRestClient
+ extends BasePaginatingAndSortingRestClient<LogbookOperationDto, ExternalHttpContext> {
+
+ private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestStreamingExternalRestClient.class);
+
+ public IngestStreamingExternalRestClient(final RestTemplate restTemplate, final String baseUrl) {
+ super(restTemplate, baseUrl);
+ }
+
+ @Override
+ public String getPathUrl() {
+ return RestApi.V1_INGEST;
+ }
+
+ @Override
+ protected Class<LogbookOperationDto> getDtoClass() {
+ return LogbookOperationDto.class;
+ }
+
+ @Override
+ protected ParameterizedTypeReference<List<LogbookOperationDto>> getDtoListClass() {
+ return new ParameterizedTypeReference<List<LogbookOperationDto>>() {
+ };
+ }
+
+ @Override
+ protected ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>> getDtoPaginatedClass() {
+ return new ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>>() {
+ };
+ }
+
+
+ public ResponseEntity<Void> streamingUpload(final ExternalHttpContext context, String fileName,
+ InputStream inputStream,
+ final String contextId,
+ final String action) {
+ LOGGER.debug("Calling upload using streaming process");
+ final UriComponentsBuilder uriBuilder =
+ UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_UPLOAD_V2);
+
+ final MultiValueMap<String, String> headersList = new HttpHeaders();
+ headersList.addAll(buildHeaders(context));
+ headersList.add(CommonConstants.X_CONTEXT_ID, contextId);
+ headersList.add(CommonConstants.X_ACTION, action);
+ headersList.add(CommonConstants.X_ORIGINAL_FILENAME_HEADER, fileName);
+ HttpHeaders headersParams = new HttpHeaders();
+ headersParams.setContentType(MediaType.APPLICATION_OCTET_STREAM);
+ headersParams.addAll(headersList);
+
+ final HttpEntity<InputStreamResource> request =
+ new HttpEntity<>(new InputStreamResource(inputStream), headersParams);
+
+ final ResponseEntity<Void> response =
+ restTemplate.exchange(uriBuilder.toUriString(), HttpMethod.POST,
+ request, Void.class);
+ LOGGER.info("The response is {}", response.toString());
+ return response;
+ }
+
+}
diff --git a/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClientFactory.java b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClientFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..9ece8976417d4b6724a07fa1fa2230cfe18bb12a
--- /dev/null
+++ b/api/api-ingest/ingest-external-client/src/main/java/fr/gouv/vitamui/ingest/external/client/IngestStreamingExternalRestClientFactory.java
@@ -0,0 +1,65 @@
+/**
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2019-2020)
+ * and the signatories of the "VITAM - Accord du Contributeur" agreement.
+ *
+ * contact@programmevitam.fr
+ *
+ * This software is a computer program whose purpose is to implement
+ * implement a digital archiving front-office system for the secure and
+ * efficient high volumetry VITAM solution.
+ *
+ * This software is governed by the CeCILL-C license under French law and
+ * abiding by the rules of distribution of free software. You can use,
+ * modify and/ or redistribute the software under the terms of the CeCILL-C
+ * license as circulated by CEA, CNRS and INRIA at the following URL
+ * "http://www.cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy,
+ * modify and redistribute granted by the license, users are provided only
+ * with a limited warranty and the software's author, the holder of the
+ * economic rights, and the successive licensors have only limited
+ * liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated
+ * with loading, using, modifying and/or developing or reproducing the
+ * software by the user in light of its specific status of free software,
+ * that may mean that it is complicated to manipulate, and that also
+ * therefore means that it is reserved for developers and experienced
+ * professionals having in-depth computer knowledge. Users are therefore
+ * encouraged to load and test the software's suitability as regards their
+ * requirements in conditions enabling the security of their systems and/or
+ * data to be ensured and, more generally, to use and operate it in the
+ * same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had
+ * knowledge of the CeCILL-C license and that you accept its terms.
+ */
+package fr.gouv.vitamui.ingest.external.client;
+
+import fr.gouv.vitamui.commons.rest.client.BaseRestClientFactory;
+import fr.gouv.vitamui.commons.rest.client.BaseStreamingRestClientFactory;
+import fr.gouv.vitamui.commons.rest.client.configuration.HttpPoolConfiguration;
+import fr.gouv.vitamui.commons.rest.client.configuration.RestClientConfiguration;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+
+/**
+ * A Rest client factory to create Streaming specialized Ingest Rest clients
+ *
+ *
+ */
+
+public class IngestStreamingExternalRestClientFactory extends BaseStreamingRestClientFactory {
+
+ public IngestStreamingExternalRestClientFactory(final RestClientConfiguration restClientConfiguration) {
+ super(restClientConfiguration);
+ }
+
+ public IngestStreamingExternalRestClientFactory(final RestClientConfiguration restClientConfiguration, final HttpPoolConfiguration httpHostConfiguration) {
+ super(restClientConfiguration, httpHostConfiguration);
+ }
+
+
+ public IngestStreamingExternalRestClient getIngestStreamingExternalRestClient() {
+ return new IngestStreamingExternalRestClient(getRestTemplate(), getBaseUrl());
+ }
+}
diff --git a/api/api-ingest/ingest-external/src/main/config/ingest-external-application-dev.yml b/api/api-ingest/ingest-external/src/main/config/ingest-external-application-dev.yml
index a001cc7ceac9b9e08b709076a5debc6d26c57bbb..05dfdc4ab945f2f6d3c8e01c189dcc41f8c861f7 100644
--- a/api/api-ingest/ingest-external/src/main/config/ingest-external-application-dev.yml
+++ b/api/api-ingest/ingest-external/src/main/config/ingest-external-application-dev.yml
@@ -11,12 +11,6 @@ spring:
enabled: false
register: false
-multipart:
- enabled: true
-
-spring.servlet.multipart.max-file-size: -1
-spring.servlet.multipart.max-request-size: -1
-
server-identity:
identityName: vitamui-dev
identityRole: ingest-external
diff --git a/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/config/ApiIngestServerConfig.java b/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/config/ApiIngestServerConfig.java
index 9b949fa80a9efe8ac29da971e24fb676ae0238e2..5494aa9393f932dbc37a0ca231297d8e04e97f03 100644
--- a/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/config/ApiIngestServerConfig.java
+++ b/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/config/ApiIngestServerConfig.java
@@ -48,6 +48,8 @@ import fr.gouv.vitamui.ingest.internal.client.IngestInternalRestClient;
import fr.gouv.vitamui.ingest.internal.client.IngestInternalRestClientFactory;
import fr.gouv.vitamui.ingest.internal.client.IngestInternalWebClient;
import fr.gouv.vitamui.ingest.internal.client.IngestInternalWebClientFactory;
+import fr.gouv.vitamui.ingest.internal.client.IngestStreamingInternalRestClient;
+import fr.gouv.vitamui.ingest.internal.client.IngestStreamingInternalRestClientFactory;
import fr.gouv.vitamui.security.client.ContextRestClient;
import fr.gouv.vitamui.security.client.SecurityRestClientFactory;
import org.springframework.boot.autoconfigure.http.HttpMessageConvertersAutoConfiguration;
@@ -55,15 +57,21 @@ import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
+import org.springframework.http.MediaType;
+import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
+
+import java.util.Arrays;
@Configuration
@Import({RestExceptionHandler.class, SwaggerConfiguration.class, HttpMessageConvertersAutoConfiguration.class})
public class ApiIngestServerConfig extends AbstractContextConfiguration {
@Bean
- public SecurityRestClientFactory securityRestClientFactory(final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
- final RestTemplateBuilder restTemplateBuilder) {
- return new SecurityRestClientFactory(apiIngestExternalApplicationProperties.getSecurityClient(), restTemplateBuilder);
+ public SecurityRestClientFactory securityRestClientFactory(
+ final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
+ final RestTemplateBuilder restTemplateBuilder) {
+ return new SecurityRestClientFactory(apiIngestExternalApplicationProperties.getSecurityClient(),
+ restTemplateBuilder);
}
@Bean
@@ -72,7 +80,19 @@ public class ApiIngestServerConfig extends AbstractContextConfiguration {
}
@Bean
- public ExternalApiAuthenticationProvider apiAuthenticationProvider(final ExternalAuthentificationService externalAuthentificationService) {
+ public MappingJackson2HttpMessageConverter customizedJacksonMessageConverter() {
+ MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter();
+ converter.setSupportedMediaTypes(
+ Arrays.asList(
+ MediaType.APPLICATION_JSON,
+ new MediaType("application", "*+json"),
+ MediaType.APPLICATION_OCTET_STREAM));
+ return converter;
+ }
+
+ @Bean
+ public ExternalApiAuthenticationProvider apiAuthenticationProvider(
+ final ExternalAuthentificationService externalAuthentificationService) {
return new ExternalApiAuthenticationProvider(externalAuthentificationService);
}
@@ -83,28 +103,46 @@ public class ApiIngestServerConfig extends AbstractContextConfiguration {
@Bean
public ExternalAuthentificationService externalAuthentificationService(final ContextRestClient contextRestClient,
- final UserInternalRestClient userInternalRestClient) {
+ final UserInternalRestClient userInternalRestClient) {
return new ExternalAuthentificationService(contextRestClient, userInternalRestClient);
}
@Bean
- public IamInternalRestClientFactory iamInternalRestClientFactory(final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
- final RestTemplateBuilder restTemplateBuilder) {
- return new IamInternalRestClientFactory(apiIngestExternalApplicationProperties.getIamInternalClient(), restTemplateBuilder);
+ public IamInternalRestClientFactory iamInternalRestClientFactory(
+ final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
+ final RestTemplateBuilder restTemplateBuilder) {
+ return new IamInternalRestClientFactory(apiIngestExternalApplicationProperties.getIamInternalClient(),
+ restTemplateBuilder);
}
@Bean
- public UserInternalRestClient userInternalRestClient(final IamInternalRestClientFactory iamInternalRestClientFactory) {
+ public UserInternalRestClient userInternalRestClient(
+ final IamInternalRestClientFactory iamInternalRestClientFactory) {
return iamInternalRestClientFactory.getUserInternalRestClient();
}
@Bean
- public IngestInternalRestClientFactory ingestInternalRestClientFactory(final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
- final RestTemplateBuilder restTemplateBuilder) {
- return new IngestInternalRestClientFactory(apiIngestExternalApplicationProperties.getIngestInternalClient(), restTemplateBuilder);
+ public IngestInternalRestClientFactory ingestInternalRestClientFactory(
+ final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
+ final RestTemplateBuilder restTemplateBuilder) {
+ return new IngestInternalRestClientFactory(apiIngestExternalApplicationProperties.getIngestInternalClient(),
+ restTemplateBuilder);
}
+ @Bean
+ public IngestStreamingInternalRestClientFactory ingestStreamingInternalRestClientFactory(
+ final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties) {
+ return new IngestStreamingInternalRestClientFactory(
+ apiIngestExternalApplicationProperties.getIngestInternalClient());
+ }
+
+
+ @Bean
+ public IngestStreamingInternalRestClient ingestStreamingInternalRestClient(
+ final IngestStreamingInternalRestClientFactory factory) {
+ return factory.getIngestStreamingInternalRestClient();
+ }
@Bean
public IngestInternalRestClient ingestInternalRestClient(final IngestInternalRestClientFactory factory) {
@@ -112,7 +150,8 @@ public class ApiIngestServerConfig extends AbstractContextConfiguration {
}
@Bean
- public IngestInternalWebClientFactory ingestInternalWebClientFactory(final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
+ public IngestInternalWebClientFactory ingestInternalWebClientFactory(
+ final ApiIngestExternalApplicationProperties apiIngestExternalApplicationProperties,
final RestTemplateBuilder restTemplateBuilder) {
return new IngestInternalWebClientFactory(apiIngestExternalApplicationProperties.getIngestInternalClient());
}
diff --git a/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/rest/IngestExternalController.java b/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/rest/IngestExternalController.java
index 62c4a663a381db87d3d2f65ee196b0747bd031ce..c0798b55e8c0a02081d94d2cb3a4b1f916598b0c 100644
--- a/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/rest/IngestExternalController.java
+++ b/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/rest/IngestExternalController.java
@@ -36,20 +36,19 @@
*/
package fr.gouv.vitamui.ingest.external.server.rest;
-import fr.gouv.vitam.common.model.RequestResponseOK;
-import fr.gouv.vitamui.common.security.SafeFileChecker;
+import fr.gouv.vitamui.common.security.SanityChecker;
import fr.gouv.vitamui.commons.api.CommonConstants;
import fr.gouv.vitamui.commons.api.ParameterChecker;
import fr.gouv.vitamui.commons.api.domain.DirectionDto;
import fr.gouv.vitamui.commons.api.domain.PaginatedValuesDto;
import fr.gouv.vitamui.commons.api.domain.ServicesData;
-import fr.gouv.vitamui.commons.api.exception.BadRequestException;
import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
import fr.gouv.vitamui.commons.vitam.api.dto.LogbookOperationDto;
import fr.gouv.vitamui.ingest.common.rest.RestApi;
import fr.gouv.vitamui.ingest.external.server.service.IngestExternalService;
import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
@@ -62,17 +61,12 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.multipart.MultipartFile;
-import reactor.core.publisher.Mono;
-import java.io.IOException;
import java.io.InputStream;
import java.util.Optional;
/**
* UI Ingest External controller
- *
- *
*/
@Api(tags = "ingest")
@RequestMapping(RestApi.V1_INGEST)
@@ -90,11 +84,14 @@ public class IngestExternalController {
}
@Secured(ServicesData.ROLE_GET_ALL_INGEST)
- @GetMapping(params = { "page", "size" })
- public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(@RequestParam final Integer page, @RequestParam final Integer size,
- @RequestParam(required = false) final Optional<String> criteria, @RequestParam(required = false) final Optional<String> orderBy,
- @RequestParam(required = false) final Optional<DirectionDto> direction) {
- LOGGER.debug("getPaginateEntities page={}, size={}, criteria={}, orderBy={}, ascendant={}", page, size, orderBy, direction);
+ @GetMapping(params = {"page", "size"})
+ public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(@RequestParam final Integer page,
+ @RequestParam final Integer size,
+ @RequestParam(required = false) final Optional<String> criteria,
+ @RequestParam(required = false) final Optional<String> orderBy,
+ @RequestParam(required = false) final Optional<DirectionDto> direction) {
+ LOGGER.debug("getPaginateEntities page={}, size={}, criteria={}, orderBy={}, ascendant={}", page, size, orderBy,
+ direction);
return ingestExternalService.getAllPaginated(page, size, criteria, orderBy, direction);
}
@@ -106,28 +103,6 @@ public class IngestExternalController {
return ingestExternalService.getOne(id);
}
- @Secured(ServicesData.ROLE_CREATE_INGEST)
- @PostMapping(value = CommonConstants.INGEST_UPLOAD, consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
- public Mono<RequestResponseOK> upload(
- @RequestHeader(value = CommonConstants.X_ACTION) final String action,
- @RequestHeader(value = CommonConstants.X_CONTEXT_ID) final String contextId,
- @RequestParam(CommonConstants.MULTIPART_FILE_PARAM_NAME) final MultipartFile file) {
- ParameterChecker
- .checkParameter("The Action and contextId are mandatory parameters : ", action, contextId);
- SafeFileChecker.checkSafeFilePath(file.getOriginalFilename());
- InputStream in = null;
- try {
- in = file.getInputStream();
- LOGGER.debug("[IngestExternalController] upload file [{}], [{}] bytes.", file.getOriginalFilename(),
- file.getInputStream().available());
- } catch (IOException e) {
- LOGGER.error("ERROR: InputStream error ", e);
- throw new BadRequestException("ERROR: InputStream writing error : ", e);
- }
-
- return ingestExternalService.upload(in, action, contextId);
- }
-
@Secured(ServicesData.ROLE_LOGBOOKS)
@GetMapping(RestApi.INGEST_REPORT_ODT + CommonConstants.PATH_ID)
public ResponseEntity<byte[]> generateODTReport(final @PathVariable("id") String id) {
@@ -135,4 +110,19 @@ public class IngestExternalController {
ParameterChecker.checkParameter("The Identifier is a mandatory parameter :", id);
return ingestExternalService.generateODTReport(id);
}
+
+ @Secured(ServicesData.ROLE_CREATE_INGEST)
+ @ApiOperation(value = "Upload an streaming SIP", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+ @PostMapping(value = CommonConstants.INGEST_UPLOAD_V2, consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+ public ResponseEntity<Void> streamingUpload(InputStream inputStream,
+ @RequestHeader(value = CommonConstants.X_ACTION) final String action,
+ @RequestHeader(value = CommonConstants.X_CONTEXT_ID) final String contextId,
+ @RequestHeader(value = CommonConstants.X_ORIGINAL_FILENAME_HEADER) final String originalFileName
+ ) {
+ LOGGER.debug("[Internal] upload file v2: {}", originalFileName);
+ ParameterChecker.checkParameter("The action and the context ID are mandatory parameters: ", action, contextId,
+ originalFileName);
+ SanityChecker.isValidFileName(originalFileName);
+ return ingestExternalService.streamingUpload(inputStream, originalFileName, contextId, action);
+ }
}
diff --git a/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/service/IngestExternalService.java b/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/service/IngestExternalService.java
index ab070920ab7d166c0f698bd1e255ecaec25bbc1d..145e65f0171187612ba32198af7e74874139ee35 100644
--- a/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/service/IngestExternalService.java
+++ b/api/api-ingest/ingest-external/src/main/java/fr/gouv/vitamui/ingest/external/server/service/IngestExternalService.java
@@ -36,7 +36,6 @@
*/
package fr.gouv.vitamui.ingest.external.server.service;
-import fr.gouv.vitam.common.model.RequestResponseOK;
import fr.gouv.vitamui.commons.api.ParameterChecker;
import fr.gouv.vitamui.commons.api.domain.DirectionDto;
import fr.gouv.vitamui.commons.api.domain.PaginatedValuesDto;
@@ -45,13 +44,12 @@ import fr.gouv.vitamui.iam.security.client.AbstractResourceClientService;
import fr.gouv.vitamui.iam.security.service.ExternalSecurityService;
import fr.gouv.vitamui.ingest.internal.client.IngestInternalRestClient;
import fr.gouv.vitamui.ingest.internal.client.IngestInternalWebClient;
+import fr.gouv.vitamui.ingest.internal.client.IngestStreamingInternalRestClient;
import lombok.Getter;
import lombok.Setter;
-
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
-import reactor.core.publisher.Mono;
import java.io.InputStream;
import java.util.Optional;
@@ -59,7 +57,6 @@ import java.util.stream.Collectors;
/**
* The service to create vitam ingest.
- *
*/
@Getter
@Setter
@@ -71,29 +68,31 @@ public class IngestExternalService extends AbstractResourceClientService<Logbook
@Autowired
private final IngestInternalWebClient ingestInternalWebClient;
+ @Autowired
+ private final IngestStreamingInternalRestClient ingestStreamingInternalRestClient;
public IngestExternalService(@Autowired IngestInternalRestClient ingestInternalRestClient,
IngestInternalWebClient ingestInternalWebClient,
- final ExternalSecurityService externalSecurityService) {
+ final ExternalSecurityService externalSecurityService,
+ final IngestStreamingInternalRestClient ingestStreamingInternalRestClient) {
super(externalSecurityService);
this.ingestInternalRestClient = ingestInternalRestClient;
this.ingestInternalWebClient = ingestInternalWebClient;
+ this.ingestStreamingInternalRestClient = ingestStreamingInternalRestClient;
}
- public Mono<RequestResponseOK> upload(InputStream in, String action, String contextId) {
- return ingestInternalWebClient.upload(getInternalHttpContext(), in, action, contextId);
- }
-
- public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(final Integer page, final Integer size, final Optional<String> criteria,
- final Optional<String> orderBy, final Optional<DirectionDto> direction) {
+ public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(final Integer page, final Integer size,
+ final Optional<String> criteria,
+ final Optional<String> orderBy, final Optional<DirectionDto> direction) {
ParameterChecker.checkPagination(size, page);
- final PaginatedValuesDto<LogbookOperationDto> result = getClient().getAllPaginated(getInternalHttpContext(), page, size, criteria, orderBy, direction);
+ final PaginatedValuesDto<LogbookOperationDto> result =
+ getClient().getAllPaginated(getInternalHttpContext(), page, size, criteria, orderBy, direction);
return new PaginatedValuesDto<>(
- result.getValues().stream().map(element -> converterToExternalDto(element)).collect(Collectors.toList()),
- result.getPageNum(),
- result.getPageSize(),
- result.isHasMore());
+ result.getValues().stream().map(element -> converterToExternalDto(element)).collect(Collectors.toList()),
+ result.getPageNum(),
+ result.getPageSize(),
+ result.isHasMore());
}
public LogbookOperationDto getOne(final String id) {
@@ -101,7 +100,7 @@ public class IngestExternalService extends AbstractResourceClientService<Logbook
}
- public ResponseEntity<byte[]> generateODTReport(String id) {
+ public ResponseEntity<byte[]> generateODTReport(String id) {
return ingestInternalRestClient.generateODTReport(getInternalHttpContext(), id);
}
@@ -109,4 +108,15 @@ public class IngestExternalService extends AbstractResourceClientService<Logbook
protected IngestInternalRestClient getClient() {
return ingestInternalRestClient;
}
+
+
+ public ResponseEntity<Void> streamingUpload(InputStream inputStream, final String originalFileName,
+ final String contextId,
+ final String action) {
+ return
+ ingestStreamingInternalRestClient
+ .streamingUpload(getInternalHttpContext(), originalFileName, inputStream, contextId,
+ action);
+ }
+
}
diff --git a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClient.java b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClient.java
index 99deb54f553847112314e30d707b07c4646d8d05..da904e49e82e0a1ffec8f125126e9082d75dc055 100644
--- a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClient.java
+++ b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClient.java
@@ -58,7 +58,8 @@ import java.util.List;
/**
* Ingest Internal REST Client.
*/
-public class IngestInternalRestClient extends BasePaginatingAndSortingRestClient<LogbookOperationDto, InternalHttpContext> {
+public class IngestInternalRestClient
+ extends BasePaginatingAndSortingRestClient<LogbookOperationDto, InternalHttpContext> {
private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestInternalRestClient.class);
@@ -71,20 +72,26 @@ public class IngestInternalRestClient extends BasePaginatingAndSortingRestClient
return RestApi.V1_INGEST;
}
- @Override protected Class<LogbookOperationDto> getDtoClass() {
+ @Override
+ protected Class<LogbookOperationDto> getDtoClass() {
return LogbookOperationDto.class;
}
- @Override protected ParameterizedTypeReference<List<LogbookOperationDto>> getDtoListClass() {
- return new ParameterizedTypeReference<List<LogbookOperationDto>>() {};
+ @Override
+ protected ParameterizedTypeReference<List<LogbookOperationDto>> getDtoListClass() {
+ return new ParameterizedTypeReference<List<LogbookOperationDto>>() {
+ };
}
- @Override protected ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>> getDtoPaginatedClass() {
- return new ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>>() {};
+ @Override
+ protected ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>> getDtoPaginatedClass() {
+ return new ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>>() {
+ };
}
- public ResponseEntity<byte[]> generateODTReport(InternalHttpContext context , String id) {
- final UriComponentsBuilder uriBuilder = UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_REPORT_ODT + CommonConstants.PATH_ID );
+ public ResponseEntity<byte[]> generateODTReport(InternalHttpContext context, String id) {
+ final UriComponentsBuilder uriBuilder =
+ UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_REPORT_ODT + CommonConstants.PATH_ID);
final HttpEntity<AuditOptions> request = new HttpEntity<>(buildHeaders(context));
return restTemplate.exchange(uriBuilder.build(id), HttpMethod.GET, request, byte[].class);
}
diff --git a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClientFactory.java b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClientFactory.java
index 1ae304bdc50d181b6416f36006e04e547c7ac799..46c4b59f8d0923953ac06b7c4244a99939c220b9 100644
--- a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClientFactory.java
+++ b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalRestClientFactory.java
@@ -43,18 +43,18 @@ import org.springframework.boot.web.client.RestTemplateBuilder;
/**
* A Rest client factory to create specialized IAM Rest clients
- *
- *
*/
public class IngestInternalRestClientFactory extends BaseRestClientFactory {
- public IngestInternalRestClientFactory(final RestClientConfiguration restClientConfiguration, final RestTemplateBuilder restTemplateBuilder) {
+ public IngestInternalRestClientFactory(final RestClientConfiguration restClientConfiguration,
+ final RestTemplateBuilder restTemplateBuilder) {
super(restClientConfiguration, restTemplateBuilder);
}
- public IngestInternalRestClientFactory(final RestClientConfiguration restClientConfiguration, final HttpPoolConfiguration httpHostConfiguration,
- final RestTemplateBuilder restTemplateBuilder) {
+ public IngestInternalRestClientFactory(final RestClientConfiguration restClientConfiguration,
+ final HttpPoolConfiguration httpHostConfiguration,
+ final RestTemplateBuilder restTemplateBuilder) {
super(restClientConfiguration, httpHostConfiguration, restTemplateBuilder);
}
diff --git a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalWebClient.java b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalWebClient.java
index 59f2b2d1b4ba2fef803fb17cf37223d154939137..2b6afb7981905b7ec526234ac91d0e0c6cc4c1e6 100644
--- a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalWebClient.java
+++ b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestInternalWebClient.java
@@ -36,29 +36,12 @@
*/
package fr.gouv.vitamui.ingest.internal.client;
-import fr.gouv.vitam.common.model.RequestResponseOK;
-import fr.gouv.vitamui.commons.api.CommonConstants;
-import fr.gouv.vitamui.commons.api.exception.BadRequestException;
-import fr.gouv.vitamui.commons.api.exception.FileOperationException;
import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
import fr.gouv.vitamui.commons.rest.client.BaseWebClient;
import fr.gouv.vitamui.commons.rest.client.InternalHttpContext;
import fr.gouv.vitamui.ingest.common.rest.RestApi;
-import org.springframework.http.HttpMethod;
-import org.springframework.util.LinkedMultiValueMap;
-import org.springframework.util.MultiValueMap;
import org.springframework.web.reactive.function.client.WebClient;
-import reactor.core.publisher.Mono;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
-import java.util.AbstractMap;
-import java.util.Optional;
/**
* External WebClient for Ingest operations.
@@ -71,35 +54,6 @@ public class IngestInternalWebClient extends BaseWebClient<InternalHttpContext>
super(webClient, baseUrl);
}
- public Mono<RequestResponseOK> upload(final InternalHttpContext context, InputStream in, final String action,
- final String contextId) {
-
- if (in == null) {
- throw new FileOperationException("There is an error in uploaded file !");
- }
-
- final Path tmpFilePath =
- Paths.get(System.getProperty(CommonConstants.VITAMUI_TEMP_DIRECTORY), context.getRequestId());
- int length = 0;
- try {
- length = in.available();
- Files.copy(in, tmpFilePath, StandardCopyOption.REPLACE_EXISTING);
- } catch (IOException e) {
- LOGGER.debug("[IngestInternalWebClient] Error writing InputStream of length [{}] to temporary path {}",
- length, tmpFilePath.toAbsolutePath());
- throw new BadRequestException("ERROR: InputStream writing error : ", e);
- }
-
- final MultiValueMap<String, String> headers = new LinkedMultiValueMap<>();
- headers.add(CommonConstants.X_CONTEXT_ID, contextId);
- headers.add(CommonConstants.X_ACTION, action);
-
- return multipartDataFromFile(getPathUrl() + CommonConstants.INGEST_UPLOAD, HttpMethod.POST, context,
- Optional.of(new AbstractMap.SimpleEntry<>(CommonConstants.MULTIPART_FILE_PARAM_NAME, tmpFilePath)),
- headers)
- .bodyToMono(RequestResponseOK.class);
- }
-
@Override
public String getPathUrl() {
return RestApi.V1_INGEST;
diff --git a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClient.java b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClient.java
new file mode 100644
index 0000000000000000000000000000000000000000..57f2d7f62b613c3af819a443f764371d3744b644
--- /dev/null
+++ b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClient.java
@@ -0,0 +1,124 @@
+/**
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2019-2020)
+ * and the signatories of the "VITAM - Accord du Contributeur" agreement.
+ *
+ * contact@programmevitam.fr
+ *
+ * This software is a computer program whose purpose is to implement
+ * implement a digital archiving front-office system for the secure and
+ * efficient high volumetry VITAM solution.
+ *
+ * This software is governed by the CeCILL-C license under French law and
+ * abiding by the rules of distribution of free software. You can use,
+ * modify and/ or redistribute the software under the terms of the CeCILL-C
+ * license as circulated by CEA, CNRS and INRIA at the following URL
+ * "http://www.cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy,
+ * modify and redistribute granted by the license, users are provided only
+ * with a limited warranty and the software's author, the holder of the
+ * economic rights, and the successive licensors have only limited
+ * liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated
+ * with loading, using, modifying and/or developing or reproducing the
+ * software by the user in light of its specific status of free software,
+ * that may mean that it is complicated to manipulate, and that also
+ * therefore means that it is reserved for developers and experienced
+ * professionals having in-depth computer knowledge. Users are therefore
+ * encouraged to load and test the software's suitability as regards their
+ * requirements in conditions enabling the security of their systems and/or
+ * data to be ensured and, more generally, to use and operate it in the
+ * same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had
+ * knowledge of the CeCILL-C license and that you accept its terms.
+ */
+package fr.gouv.vitamui.ingest.internal.client;
+
+
+import fr.gouv.vitam.common.model.AuditOptions;
+import fr.gouv.vitamui.commons.api.CommonConstants;
+import fr.gouv.vitamui.commons.api.domain.PaginatedValuesDto;
+import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
+import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
+import fr.gouv.vitamui.commons.rest.client.BasePaginatingAndSortingRestClient;
+import fr.gouv.vitamui.commons.rest.client.InternalHttpContext;
+import fr.gouv.vitamui.commons.vitam.api.dto.LogbookOperationDto;
+import fr.gouv.vitamui.ingest.common.rest.RestApi;
+import org.springframework.core.ParameterizedTypeReference;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.HttpHeaders;
+import org.springframework.http.HttpMethod;
+import org.springframework.http.MediaType;
+import org.springframework.http.ResponseEntity;
+import org.springframework.util.MultiValueMap;
+import org.springframework.web.client.RestTemplate;
+import org.springframework.web.util.UriComponentsBuilder;
+
+import java.io.InputStream;
+import java.util.List;
+
+/**
+ * Ingest Streaming Internal REST Client.
+ */
+public class IngestStreamingInternalRestClient
+ extends BasePaginatingAndSortingRestClient<LogbookOperationDto, InternalHttpContext> {
+
+ private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestStreamingInternalRestClient.class);
+
+ public IngestStreamingInternalRestClient(final RestTemplate restTemplate, final String baseUrl) {
+ super(restTemplate, baseUrl);
+ }
+
+ @Override
+ public String getPathUrl() {
+ return RestApi.V1_INGEST;
+ }
+
+ @Override
+ protected Class<LogbookOperationDto> getDtoClass() {
+ return LogbookOperationDto.class;
+ }
+
+ @Override
+ protected ParameterizedTypeReference<List<LogbookOperationDto>> getDtoListClass() {
+ return new ParameterizedTypeReference<List<LogbookOperationDto>>() {
+ };
+ }
+
+ @Override
+ protected ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>> getDtoPaginatedClass() {
+ return new ParameterizedTypeReference<PaginatedValuesDto<LogbookOperationDto>>() {
+ };
+ }
+
+ public ResponseEntity<Void> streamingUpload(final InternalHttpContext context, String originalFileName,
+ InputStream inputStream,
+ final String contextId,
+ final String action) {
+ LOGGER.debug("Calling upload using streaming process");
+ final UriComponentsBuilder uriBuilder =
+ UriComponentsBuilder.fromHttpUrl(getUrl() + RestApi.INGEST_UPLOAD_V2);
+
+ final MultiValueMap<String, String> headersList = new HttpHeaders();
+ headersList.addAll(buildHeaders(context));
+ headersList.add(CommonConstants.X_CONTEXT_ID, contextId);
+ headersList.add(CommonConstants.X_ACTION, action);
+ headersList.add(CommonConstants.X_ORIGINAL_FILENAME_HEADER, originalFileName);
+
+ HttpHeaders headersParams = new HttpHeaders();
+ headersParams.setContentType(MediaType.APPLICATION_OCTET_STREAM);
+ headersParams.addAll(headersList);
+
+ final HttpEntity<InputStreamResource> request =
+ new HttpEntity<>(new InputStreamResource(inputStream), headersParams);
+
+ final ResponseEntity<Void> response =
+ restTemplate.exchange(uriBuilder.toUriString(), HttpMethod.POST,
+ request, Void.class);
+ LOGGER.info("The response on ingest is {} ", response.toString());
+ return response;
+ }
+}
diff --git a/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClientFactory.java b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClientFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..2bef50b288c1534059d044f5bc4803e1afd35846
--- /dev/null
+++ b/api/api-ingest/ingest-internal-client/src/main/java/fr/gouv/vitamui/ingest/internal/client/IngestStreamingInternalRestClientFactory.java
@@ -0,0 +1,64 @@
+/**
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2019-2020)
+ * and the signatories of the "VITAM - Accord du Contributeur" agreement.
+ *
+ * contact@programmevitam.fr
+ *
+ * This software is a computer program whose purpose is to implement
+ * implement a digital archiving front-office system for the secure and
+ * efficient high volumetry VITAM solution.
+ *
+ * This software is governed by the CeCILL-C license under French law and
+ * abiding by the rules of distribution of free software. You can use,
+ * modify and/ or redistribute the software under the terms of the CeCILL-C
+ * license as circulated by CEA, CNRS and INRIA at the following URL
+ * "http://www.cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy,
+ * modify and redistribute granted by the license, users are provided only
+ * with a limited warranty and the software's author, the holder of the
+ * economic rights, and the successive licensors have only limited
+ * liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated
+ * with loading, using, modifying and/or developing or reproducing the
+ * software by the user in light of its specific status of free software,
+ * that may mean that it is complicated to manipulate, and that also
+ * therefore means that it is reserved for developers and experienced
+ * professionals having in-depth computer knowledge. Users are therefore
+ * encouraged to load and test the software's suitability as regards their
+ * requirements in conditions enabling the security of their systems and/or
+ * data to be ensured and, more generally, to use and operate it in the
+ * same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had
+ * knowledge of the CeCILL-C license and that you accept its terms.
+ */
+package fr.gouv.vitamui.ingest.internal.client;
+
+import fr.gouv.vitamui.commons.rest.client.BaseRestClientFactory;
+import fr.gouv.vitamui.commons.rest.client.BaseStreamingRestClientFactory;
+import fr.gouv.vitamui.commons.rest.client.configuration.HttpPoolConfiguration;
+import fr.gouv.vitamui.commons.rest.client.configuration.RestClientConfiguration;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+
+/**
+ * A Rest client factory to create specialized IAM Rest clients
+ *
+ *
+ */
+
+public class IngestStreamingInternalRestClientFactory extends BaseStreamingRestClientFactory {
+
+ public IngestStreamingInternalRestClientFactory(final RestClientConfiguration restClientConfiguration) {
+ super(restClientConfiguration);
+ }
+
+ public IngestStreamingInternalRestClientFactory(final RestClientConfiguration restClientConfiguration, final HttpPoolConfiguration httpHostConfiguration) {
+ super(restClientConfiguration, httpHostConfiguration);
+ }
+
+ public IngestStreamingInternalRestClient getIngestStreamingInternalRestClient() {
+ return new IngestStreamingInternalRestClient(getRestTemplate(), getBaseUrl());
+ }
+}
diff --git a/api/api-ingest/ingest-internal/src/main/config/ingest-internal-application-dev.yml b/api/api-ingest/ingest-internal/src/main/config/ingest-internal-application-dev.yml
index 14160e8ec79fe4e42eebf549c71a35b015b6b1c5..a2b8f121de64a4d53b5319a26f4800289246419b 100644
--- a/api/api-ingest/ingest-internal/src/main/config/ingest-internal-application-dev.yml
+++ b/api/api-ingest/ingest-internal/src/main/config/ingest-internal-application-dev.yml
@@ -15,12 +15,6 @@ spring:
mongodb:
uri: mongodb://mongod_dbuser_iam:mongod_dbpwd_iam@localhost:27018/iam?connectTimeoutMS=2000
-multipart:
- enabled: true
-
-spring.servlet.multipart.max-file-size: -1
-spring.servlet.multipart.max-request-size: -1
-
server-identity:
identityName: vitamui-dev
identityRole: ingest-internal
diff --git a/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/config/ApiIngestInternalServerConfig.java b/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/config/ApiIngestInternalServerConfig.java
index 317e62331c6f2f452223be10f3e3a96df9cf249a..739613eb1f17495a0494730be5dcaecad102dc90 100644
--- a/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/config/ApiIngestInternalServerConfig.java
+++ b/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/config/ApiIngestInternalServerConfig.java
@@ -36,6 +36,7 @@
*/
package fr.gouv.vitamui.ingest.internal.server.config;
+import com.fasterxml.jackson.databind.ObjectMapper;
import fr.gouv.vitam.ingest.external.client.IngestExternalClient;
import fr.gouv.vitamui.commons.api.application.AbstractContextConfiguration;
import fr.gouv.vitamui.commons.mongo.config.MongoConfig;
@@ -61,11 +62,14 @@ import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Import;
+import org.springframework.http.MediaType;
+import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.Arrays;
@Configuration
-@Import({RestExceptionHandler.class, MongoConfig.class, SwaggerConfiguration.class, WebSecurityConfig.class, VitamAccessConfig.class, VitamIngestConfig.class,
+@Import({RestExceptionHandler.class, MongoConfig.class, SwaggerConfiguration.class, WebSecurityConfig.class,
+ VitamAccessConfig.class, VitamIngestConfig.class,
VitamAdministrationConfig.class})
public class ApiIngestInternalServerConfig extends AbstractContextConfiguration {
@@ -83,17 +87,31 @@ public class ApiIngestInternalServerConfig extends AbstractContextConfiguration
}
@Bean
- public InternalApiAuthenticationProvider internalApiAuthenticationProvider(final InternalAuthentificationService internalAuthentificationService) {
+ public MappingJackson2HttpMessageConverter customizedJacksonMessageConverter() {
+ MappingJackson2HttpMessageConverter converter = new MappingJackson2HttpMessageConverter();
+ converter.setSupportedMediaTypes(
+ Arrays.asList(
+ MediaType.APPLICATION_JSON,
+ new MediaType("application", "*+json"),
+ MediaType.APPLICATION_OCTET_STREAM));
+ return converter;
+ }
+
+ @Bean
+ public InternalApiAuthenticationProvider internalApiAuthenticationProvider(
+ final InternalAuthentificationService internalAuthentificationService) {
return new InternalApiAuthenticationProvider(internalAuthentificationService);
}
@Bean
- public UserInternalRestClient userInternalRestClient(final IamInternalRestClientFactory iamInternalRestClientFactory) {
+ public UserInternalRestClient userInternalRestClient(
+ final IamInternalRestClientFactory iamInternalRestClientFactory) {
return iamInternalRestClientFactory.getUserInternalRestClient();
}
@Bean
- public InternalAuthentificationService internalAuthentificationService(final UserInternalRestClient userInternalRestClient) {
+ public InternalAuthentificationService internalAuthentificationService(
+ final UserInternalRestClient userInternalRestClient) {
return new InternalAuthentificationService(userInternalRestClient);
}
@@ -101,8 +119,10 @@ public class ApiIngestInternalServerConfig extends AbstractContextConfiguration
public InternalSecurityService securityService() {
return new InternalSecurityService();
}
+
@Bean
- public CustomerInternalRestClient customerInternalRestClient(final IamInternalRestClientFactory iamInternalRestClientFactory) {
+ public CustomerInternalRestClient customerInternalRestClient(
+ final IamInternalRestClientFactory iamInternalRestClientFactory) {
return iamInternalRestClientFactory.getCustomerInternalRestClient();
}
@@ -113,14 +133,15 @@ public class ApiIngestInternalServerConfig extends AbstractContextConfiguration
@Bean
public IngestInternalService ingestInternalService(
- final InternalSecurityService internalSecurityService,
- final LogbookService logbookService,
- final ObjectMapper objectMapper,
- final IngestExternalClient ingestExternalClient,
- final IngestService ingestService,
- final CustomerInternalRestClient customerInternalRestClient,
- final IngestGeneratorODTFile ingestGeneratorODTFile) {
- return new IngestInternalService(internalSecurityService, logbookService, objectMapper, ingestExternalClient, ingestService,
+ final InternalSecurityService internalSecurityService,
+ final LogbookService logbookService,
+ final ObjectMapper objectMapper,
+ final IngestExternalClient ingestExternalClient,
+ final IngestService ingestService,
+ final CustomerInternalRestClient customerInternalRestClient,
+ final IngestGeneratorODTFile ingestGeneratorODTFile) {
+ return new IngestInternalService(internalSecurityService, logbookService, objectMapper, ingestExternalClient,
+ ingestService,
customerInternalRestClient, ingestGeneratorODTFile);
}
}
diff --git a/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/rest/IngestInternalController.java b/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/rest/IngestInternalController.java
index be8f72b06346cceeb8e5f5dd33c91d16e910a0ee..bd872428f4831d32d3c9d1489870303000da3f30 100644
--- a/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/rest/IngestInternalController.java
+++ b/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/rest/IngestInternalController.java
@@ -27,7 +27,6 @@
package fr.gouv.vitamui.ingest.internal.server.rest;
import fr.gouv.vitam.common.client.VitamContext;
-import fr.gouv.vitam.common.model.RequestResponseOK;
import fr.gouv.vitam.ingest.external.api.exception.IngestExternalException;
import fr.gouv.vitamui.common.security.SanityChecker;
import fr.gouv.vitamui.commons.api.CommonConstants;
@@ -37,15 +36,14 @@ import fr.gouv.vitamui.commons.api.domain.PaginatedValuesDto;
import fr.gouv.vitamui.commons.api.exception.IngestFileGenerationException;
import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
-import fr.gouv.vitamui.iam.security.service.InternalSecurityService;
import fr.gouv.vitamui.commons.vitam.api.dto.LogbookOperationDto;
+import fr.gouv.vitamui.iam.security.service.InternalSecurityService;
import fr.gouv.vitamui.ingest.common.rest.RestApi;
import fr.gouv.vitamui.ingest.internal.server.service.IngestInternalService;
-
import io.swagger.annotations.Api;
+import io.swagger.annotations.ApiOperation;
import lombok.Getter;
import lombok.Setter;
-
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
@@ -57,9 +55,9 @@ import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
-import org.springframework.web.multipart.MultipartFile;
import java.io.IOException;
+import java.io.InputStream;
import java.net.URISyntaxException;
import java.util.Optional;
@@ -77,16 +75,21 @@ public class IngestInternalController {
private InternalSecurityService securityService;
@Autowired
- public IngestInternalController(final IngestInternalService ingestInternalService, final InternalSecurityService securityService) {
+ public IngestInternalController(final IngestInternalService ingestInternalService,
+ final InternalSecurityService securityService) {
this.ingestInternalService = ingestInternalService;
this.securityService = securityService;
}
@GetMapping(params = {"page", "size"})
- public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(@RequestParam final Integer page, @RequestParam final Integer size,
- @RequestParam(required = false) final Optional<String> criteria, @RequestParam(required = false) final Optional<String> orderBy,
- @RequestParam(required = false) final Optional<DirectionDto> direction) {
- LOGGER.debug("getPaginateEntities page={}, size={}, criteria={}, orderBy={}, ascendant={}", page, size, criteria, orderBy, direction);
+ public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(@RequestParam final Integer page,
+ @RequestParam final Integer size,
+ @RequestParam(required = false) final Optional<String> criteria,
+ @RequestParam(required = false) final Optional<String> orderBy,
+ @RequestParam(required = false) final Optional<DirectionDto> direction) {
+ LOGGER
+ .debug("getPaginateEntities page={}, size={}, criteria={}, orderBy={}, ascendant={}", page, size, criteria,
+ orderBy, direction);
final VitamContext vitamContext = securityService.buildVitamContext(securityService.getTenantIdentifier());
return ingestInternalService.getAllPaginated(page, size, orderBy, direction, vitamContext, criteria);
}
@@ -99,31 +102,34 @@ public class IngestInternalController {
return ingestInternalService.getOne(vitamContext, id);
}
- @PostMapping(value = CommonConstants.INGEST_UPLOAD, consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
- public RequestResponseOK upload(
- @RequestHeader(value = CommonConstants.X_ACTION) final String action,
- @RequestHeader(value = CommonConstants.X_CONTEXT_ID) final String contextId,
- @RequestParam(CommonConstants.MULTIPART_FILE_PARAM_NAME) final MultipartFile path)
- throws IngestExternalException {
- LOGGER.debug("[Internal] upload file : {}", path.getOriginalFilename());
- ParameterChecker.checkParameter("The action and the context ID are mandatory parameters: ", action, contextId);
- SanityChecker.isValidFileName(path.getOriginalFilename());
- return ingestInternalService.upload(path, contextId, action);
- }
-
@GetMapping(RestApi.INGEST_REPORT_ODT + CommonConstants.PATH_ID)
public ResponseEntity<byte[]> generateODTReport(final @PathVariable("id") String id)
throws IngestFileGenerationException {
final VitamContext vitamContext = securityService.buildVitamContext(securityService.getTenantIdentifier());
- try {
- LOGGER.debug("export ODT report for operation with id :{}", id);
- ParameterChecker.checkParameter("Identifier is mandatory : ", id);
- byte[] response = this.ingestInternalService.generateODTReport(vitamContext, id);
- return new ResponseEntity<>(response, HttpStatus.OK);
- }
- catch(IOException | URISyntaxException | IngestFileGenerationException e) {
- LOGGER.error("Error with generating Report : {} " , e.getMessage());
+ try {
+ LOGGER.debug("export ODT report for operation with id :{}", id);
+ ParameterChecker.checkParameter("Identifier is mandatory : ", id);
+ byte[] response = this.ingestInternalService.generateODTReport(vitamContext, id);
+ return new ResponseEntity<>(response, HttpStatus.OK);
+ } catch (IOException | URISyntaxException | IngestFileGenerationException e) {
+ LOGGER.error("Error with generating Report : {} ", e.getMessage());
throw new IngestFileGenerationException("Unable to generate the ingest report " + e);
- }
+ }
+ }
+
+ @ApiOperation(value = "Upload an SIP", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+ @PostMapping(value = CommonConstants.INGEST_UPLOAD_V2, consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+ public void streamingUpload(
+ InputStream inputStream,
+ @RequestHeader(value = CommonConstants.X_ACTION) final String action,
+ @RequestHeader(value = CommonConstants.X_CONTEXT_ID) final String contextId,
+ @RequestHeader(value = CommonConstants.X_ORIGINAL_FILENAME_HEADER) final String originalFileName
+ )
+ throws IngestExternalException {
+ LOGGER.debug("[Internal] upload file v2: {}", originalFileName);
+ ParameterChecker.checkParameter("The action and the context ID are mandatory parameters: ", action, contextId,
+ originalFileName);
+ SanityChecker.isValidFileName(originalFileName);
+ ingestInternalService.streamingUpload(inputStream, contextId, action);
}
}
diff --git a/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/service/IngestInternalService.java b/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/service/IngestInternalService.java
index 31840351bec10aa87052d2d7f8847ec2c9e8ce9f..90e79ab6701a0586d3c6c11ba62e6c6c342985bd 100644
--- a/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/service/IngestInternalService.java
+++ b/api/api-ingest/ingest-internal/src/main/java/fr/gouv/vitamui/ingest/internal/server/service/IngestInternalService.java
@@ -57,7 +57,6 @@ import fr.gouv.vitamui.iam.security.service.InternalSecurityService;
import fr.gouv.vitamui.ingest.common.dsl.VitamQueryHelper;
import fr.gouv.vitamui.ingest.common.dto.ArchiveUnitDto;
import fr.gouv.vitamui.ingest.internal.server.rest.IngestInternalController;
-
import org.odftoolkit.simple.TextDocument;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.io.InputStreamResource;
@@ -65,8 +64,8 @@ import org.springframework.core.io.Resource;
import org.springframework.web.multipart.MultipartFile;
import org.w3c.dom.Document;
-import java.io.ByteArrayOutputStream;
import javax.ws.rs.core.Response;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URISyntaxException;
@@ -77,8 +76,6 @@ import java.util.Optional;
/**
* Ingest Internal service communication with VITAM.
- *
- *
*/
public class IngestInternalService {
private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestInternalController.class);
@@ -103,8 +100,7 @@ public class IngestInternalService {
final LogbookService logbookService, final ObjectMapper objectMapper,
final IngestExternalClient ingestExternalClient, final IngestService ingestService,
final CustomerInternalRestClient customerInternalRestClient,
- final IngestGeneratorODTFile ingestGeneratorODTFile)
- {
+ final IngestGeneratorODTFile ingestGeneratorODTFile) {
this.internalSecurityService = internalSecurityService;
this.ingestExternalClient = ingestExternalClient;
this.logbookService = logbookService;
@@ -123,14 +119,13 @@ public class IngestInternalService {
RequestResponse<Void> ingestResponse = null;
try {
- LOGGER.info("Upload EvIdAppSession : {} " , vitamContext.getApplicationSessionId());
+ LOGGER.info("Upload EvIdAppSession : {} ", vitamContext.getApplicationSessionId());
ingestResponse = ingestService.ingest(vitamContext, path.getInputStream(), contextId, action);
LOGGER.info("The recieved stream size : " + path.getInputStream().available() + " is sent to Vitam");
- if(ingestResponse.isOk()) {
+ if (ingestResponse.isOk()) {
LOGGER.debug("Ingest passed successfully : " + ingestResponse.toString());
- }
- else {
+ } else {
LOGGER.debug("Ingest failed with status : " + ingestResponse.getHttpCode());
}
} catch (IOException | IngestExternalException e) {
@@ -148,7 +143,7 @@ public class IngestInternalService {
Map<String, Object> vitamCriteria = new HashMap<>();
JsonNode query;
try {
- LOGGER.info(" All ingests EvIdAppSession : {} " , vitamContext.getApplicationSessionId());
+ LOGGER.info(" All ingests EvIdAppSession : {} ", vitamContext.getApplicationSessionId());
if (criteria.isPresent()) {
TypeReference<HashMap<String, Object>> typRef = new TypeReference<HashMap<String, Object>>() {
};
@@ -173,13 +168,14 @@ public class IngestInternalService {
final RequestResponse<LogbookOperation> requestResponse;
try {
- LOGGER.info("Ingest EvIdAppSession : {} " , vitamContext.getApplicationSessionId());
+ LOGGER.info("Ingest EvIdAppSession : {} ", vitamContext.getApplicationSessionId());
requestResponse = logbookService.selectOperationbyId(id, vitamContext);
LOGGER.debug("One Ingest Response: {}: ", requestResponse);
- final LogbookOperationsResponseDto logbookOperationDtos = objectMapper.treeToValue(requestResponse.toJsonNode(), LogbookOperationsResponseDto.class);
+ final LogbookOperationsResponseDto logbookOperationDtos =
+ objectMapper.treeToValue(requestResponse.toJsonNode(), LogbookOperationsResponseDto.class);
List<LogbookOperationDto> singleLogbookOperationDto =
IngestConverter.convertVitamsToDtos(logbookOperationDtos.getResults());
@@ -194,7 +190,7 @@ public class IngestInternalService {
private LogbookOperationsResponseDto findAll(VitamContext vitamContext, JsonNode query) {
final RequestResponse<LogbookOperation> requestResponse;
try {
- LOGGER.info("All Ingest EvIdAppSession : {} " , vitamContext.getApplicationSessionId());
+ LOGGER.info("All Ingest EvIdAppSession : {} ", vitamContext.getApplicationSessionId());
requestResponse = logbookService.selectOperations(query, vitamContext);
LOGGER.debug("Response: {}: ", requestResponse);
@@ -256,29 +252,32 @@ public class IngestInternalService {
try {
Document atr = ingestGeneratorODTFile.convertStringToXMLDocument(getAtrAsString(vitamContext, id));
- Document manifest = ingestGeneratorODTFile.convertStringToXMLDocument(getManifestAsString(vitamContext, id));
+ Document manifest =
+ ingestGeneratorODTFile.convertStringToXMLDocument(getManifestAsString(vitamContext, id));
TextDocument document;
try {
document = TextDocument.newTextDocument();
} catch (Exception e) {
- LOGGER.error("Error to initialize the document : {} " , e.getMessage());
- throw new IngestFileGenerationException("Error to initialize the document : {} " , e);
+ LOGGER.error("Error to initialize the document : {} ", e.getMessage());
+ throw new IngestFileGenerationException("Error to initialize the document : {} ", e);
}
- if(myCustomer.isHasCustomGraphicIdentity()) {
- customerLogo = customerInternalRestClient.getLogo(internalSecurityService.getHttpContext(), myCustomer.getId(), AttachmentType.HEADER).getBody();
+ if (myCustomer.isHasCustomGraphicIdentity()) {
+ customerLogo = customerInternalRestClient
+ .getLogo(internalSecurityService.getHttpContext(), myCustomer.getId(), AttachmentType.HEADER)
+ .getBody();
}
- List<ArchiveUnitDto> archiveUnitDtoList = ingestGeneratorODTFile.getValuesForDynamicTable(atr,manifest);
+ List<ArchiveUnitDto> archiveUnitDtoList = ingestGeneratorODTFile.getValuesForDynamicTable(atr, manifest);
- ingestGeneratorODTFile.generateDocumentHeader(document,myCustomer,customerLogo);
+ ingestGeneratorODTFile.generateDocumentHeader(document, myCustomer, customerLogo);
ingestGeneratorODTFile.generateFirstTitle(document);
- ingestGeneratorODTFile.generateServicesTable(document,manifest);
+ ingestGeneratorODTFile.generateServicesTable(document, manifest);
- ingestGeneratorODTFile.generateDepositDataTable(document,manifest,archiveUnitDtoList);
+ ingestGeneratorODTFile.generateDepositDataTable(document, manifest, archiveUnitDtoList);
- ingestGeneratorODTFile.generateOperationDataTable(document,manifest,id);
+ ingestGeneratorODTFile.generateOperationDataTable(document, manifest, id);
ingestGeneratorODTFile.generateResponsibleSignatureTable(document);
@@ -286,24 +285,44 @@ public class IngestInternalService {
ingestGeneratorODTFile.generateSecondtTitle(document);
- ingestGeneratorODTFile.generateArchiveUnitDetailsTable(document,archiveUnitDtoList);
+ ingestGeneratorODTFile.generateArchiveUnitDetailsTable(document, archiveUnitDtoList);
- LOGGER.info("Generate ODT Report EvIdAppSession : {} " , vitamContext.getApplicationSessionId());
+ LOGGER.info("Generate ODT Report EvIdAppSession : {} ", vitamContext.getApplicationSessionId());
ByteArrayOutputStream result = new ByteArrayOutputStream();
try {
document.save(result);
} catch (Exception e) {
- LOGGER.error("Error to save the document : {} " , e.getMessage());
- throw new IngestFileGenerationException("Error to save the document : {} " , e);
+ LOGGER.error("Error to save the document : {} ", e.getMessage());
+ throw new IngestFileGenerationException("Error to save the document : {} ", e);
}
return result.toByteArray();
} catch (IOException | URISyntaxException | IngestFileGenerationException e) {
- LOGGER.error("Error with generating Report : {} " , e.getMessage());
- throw new IngestFileGenerationException("Unable to generate the ingest report ", e) ;
+ LOGGER.error("Error with generating Report : {} ", e.getMessage());
+ throw new IngestFileGenerationException("Unable to generate the ingest report ", e);
}
}
+ public void streamingUpload(InputStream inputStream, String contextId, String action)
+ throws IngestExternalException {
+ RequestResponse<Void> ingestResponse;
+ try {
+ final VitamContext vitamContext =
+ internalSecurityService.buildVitamContext(internalSecurityService.getTenantIdentifier());
+ ingestResponse =
+ ingestExternalClient.ingest(vitamContext, inputStream, contextId, action);
+
+ if (ingestResponse.isOk()) {
+ LOGGER.debug("Ingest passed successfully : " + ingestResponse.toString());
+ } else {
+ LOGGER.debug("Ingest failed with status : " + ingestResponse.getHttpCode());
+
+ }
+ } catch (Exception e) {
+ LOGGER.debug("Error sending upload to vitam ", e);
+ throw new IngestExternalException(e);
+ }
+ }
}
diff --git a/commons/commons-api/src/main/java/fr/gouv/vitamui/commons/api/CommonConstants.java b/commons/commons-api/src/main/java/fr/gouv/vitamui/commons/api/CommonConstants.java
index 0c41fb04347aaae8dfb42fe758e6ee8a75360247..3cd59ef99b988bbc16fe0d8866a270d2b2f099ef 100644
--- a/commons/commons-api/src/main/java/fr/gouv/vitamui/commons/api/CommonConstants.java
+++ b/commons/commons-api/src/main/java/fr/gouv/vitamui/commons/api/CommonConstants.java
@@ -313,6 +313,7 @@ public class CommonConstants {
public static final String MULTIPART_FILE_PARAM_NAME = "uploadedFile";
public static final String INGEST_UPLOAD = "/upload";
+ public static final String INGEST_UPLOAD_V2 = "/upload-v2";
public static final String X_ACTION = "X-Action";
public static final String X_CONTEXT_ID = "X-Context-Id";
public static final String X_SIZE_TOTAL = "X-Size-Total";
diff --git a/commons/commons-rest/src/main/java/fr/gouv/vitamui/commons/rest/client/BaseStreamingRestClientFactory.java b/commons/commons-rest/src/main/java/fr/gouv/vitamui/commons/rest/client/BaseStreamingRestClientFactory.java
new file mode 100644
index 0000000000000000000000000000000000000000..0d621dae64521a04d42b72bc0ed26433cba808b1
--- /dev/null
+++ b/commons/commons-rest/src/main/java/fr/gouv/vitamui/commons/rest/client/BaseStreamingRestClientFactory.java
@@ -0,0 +1,231 @@
+/**
+ * Copyright French Prime minister Office/SGMAP/DINSIC/Vitam Program (2019-2020)
+ * and the signatories of the "VITAM - Accord du Contributeur" agreement.
+ *
+ * contact@programmevitam.fr
+ *
+ * This software is a computer program whose purpose is to implement
+ * implement a digital archiving front-office system for the secure and
+ * efficient high volumetry VITAM solution.
+ *
+ * This software is governed by the CeCILL-C license under French law and
+ * abiding by the rules of distribution of free software. You can use,
+ * modify and/ or redistribute the software under the terms of the CeCILL-C
+ * license as circulated by CEA, CNRS and INRIA at the following URL
+ * "http://www.cecill.info".
+ *
+ * As a counterpart to the access to the source code and rights to copy,
+ * modify and redistribute granted by the license, users are provided only
+ * with a limited warranty and the software's author, the holder of the
+ * economic rights, and the successive licensors have only limited
+ * liability.
+ *
+ * In this respect, the user's attention is drawn to the risks associated
+ * with loading, using, modifying and/or developing or reproducing the
+ * software by the user in light of its specific status of free software,
+ * that may mean that it is complicated to manipulate, and that also
+ * therefore means that it is reserved for developers and experienced
+ * professionals having in-depth computer knowledge. Users are therefore
+ * encouraged to load and test the software's suitability as regards their
+ * requirements in conditions enabling the security of their systems and/or
+ * data to be ensured and, more generally, to use and operate it in the
+ * same conditions as regards security.
+ *
+ * The fact that you are presently reading this means that you have had
+ * knowledge of the CeCILL-C license and that you accept its terms.
+ */
+package fr.gouv.vitamui.commons.rest.client;
+
+import fr.gouv.vitamui.commons.api.exception.ApplicationServerException;
+import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
+import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
+import fr.gouv.vitamui.commons.rest.client.configuration.HttpPoolConfiguration;
+import fr.gouv.vitamui.commons.rest.client.configuration.RestClientConfiguration;
+import fr.gouv.vitamui.commons.rest.client.configuration.SSLConfiguration;
+import fr.gouv.vitamui.commons.rest.util.RestUtils;
+import org.apache.http.HttpHost;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.config.Registry;
+import org.apache.http.config.RegistryBuilder;
+import org.apache.http.conn.routing.HttpRoute;
+import org.apache.http.conn.socket.ConnectionSocketFactory;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.client.BufferingClientHttpRequestFactory;
+import org.springframework.http.client.ClientHttpRequestInterceptor;
+import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.util.Assert;
+import org.springframework.util.ResourceUtils;
+import org.springframework.web.client.RestTemplate;
+
+import javax.net.ssl.HostnameVerifier;
+import javax.net.ssl.SSLContext;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.KeyManagementException;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ * A rest client factory to create each domain specific REST client. The http connection is configured by the
+ * RestClientConfiguration object. The factory implements a connection pool configured by the HttpPoolConfiguration
+ * object and handles SSL via x509 certificates.
+ *
+ *
+ */
+
+public class BaseStreamingRestClientFactory implements RestClientFactory {
+
+ private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(BaseStreamingRestClientFactory.class);
+
+ private final RestTemplate restTemplate;
+
+ private final String baseUrl;
+
+ protected int connectTimeout = 500000;
+
+ protected int connectionRequestTimeout = 500000;
+
+ protected int socketTimeout = 500000;
+
+ public BaseStreamingRestClientFactory(final RestClientConfiguration restClientConfiguration) {
+ this(restClientConfiguration, null);
+ }
+
+ public BaseStreamingRestClientFactory(final RestClientConfiguration restClientConfig, final HttpPoolConfiguration httpPoolConfig) {
+ Assert.notNull(restClientConfig, "Rest client configuration must be specified");
+
+ final boolean useSSL = restClientConfig.isSecure();
+ baseUrl = RestUtils.getScheme(useSSL) + restClientConfig.getServerHost() + ":" + restClientConfig.getServerPort();
+
+ HttpPoolConfiguration myPoolConfig = httpPoolConfig;
+
+ // configure the pool from the restClientConfig if the value of poolMaxTotal is positive
+ if(restClientConfig.getPoolMaxTotal() >= 0) {
+ myPoolConfig = new HttpPoolConfiguration();
+ myPoolConfig.setMaxTotal(restClientConfig.getPoolMaxTotal());
+ myPoolConfig.setMaxPerRoute(restClientConfig.getPoolMaxPerRoute());
+ }
+
+ final Registry<ConnectionSocketFactory> csfRegistry = useSSL ? buildRegistry(restClientConfig.getSslConfiguration()) : null;
+ final PoolingHttpClientConnectionManager connectionManager = buildConnectionManager(myPoolConfig, csfRegistry);
+ final RequestConfig requestConfig = buildRequestConfig();
+
+
+ final CloseableHttpClient httpClient = HttpClientBuilder.create()
+ .setConnectionManager(connectionManager)
+ .setDefaultRequestConfig(requestConfig)
+ .build();
+
+ HttpComponentsClientHttpRequestFactory requestFactory = new HttpComponentsClientHttpRequestFactory(httpClient);
+ requestFactory.setBufferRequestBody(false);
+ restTemplate = new RestTemplate(requestFactory);
+ restTemplate.setErrorHandler(new ErrorHandler());
+ }
+
+ /*
+ * Create an SSLContext that uses client.p12 as the client certificate
+ * and the truststore.jks as the trust material (trusted CA certificates).
+ * Then create SSLConnectionSocketFactory to register with the HTTPS protocol.
+ */
+ private Registry<ConnectionSocketFactory> buildRegistry(final SSLConfiguration sslConfiguration) {
+ if (sslConfiguration == null) {
+ throw new ApplicationServerException("SSL Configuration is not defined. Unable to configure the SSLConnection");
+ }
+
+ final SSLConfiguration.CertificateStoreConfiguration ks = sslConfiguration.getKeystore();
+ final SSLConfiguration.CertificateStoreConfiguration ts = sslConfiguration.getTruststore();
+
+ SSLContext sslContext = null;
+ try {
+
+ final SSLContextBuilder sslContextBuilder = SSLContextBuilder.create();
+
+ if (ks != null) {
+ final KeyStore keyStore = loadPkcs(ks.getType(), ks.getKeyPath(), ks.getKeyPassword().toCharArray());
+ sslContextBuilder.loadKeyMaterial(keyStore, ks.getKeyPassword().toCharArray());
+ }
+
+ sslContext = sslContextBuilder.loadTrustMaterial(new File(ts.getKeyPath()), ts.getKeyPassword().toCharArray()).setProtocol("TLS")
+ .setSecureRandom(new java.security.SecureRandom()).build();
+ }
+ catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException | CertificateException | IOException | UnrecoverableKeyException e) {
+ LOGGER.error("Unable to build the Registry<ConnectionSocketFactory>.", e);
+ LOGGER.error("KeyPath: " + sslConfiguration.getKeystore().getKeyPath());
+
+ throw new ApplicationServerException(e);
+ }
+
+ final HostnameVerifier hostnameVerifier = sslConfiguration.isHostnameVerification() ? null : TrustAllHostnameVerifier.INSTANCE;
+ final SSLConnectionSocketFactory sslFactory = new SSLConnectionSocketFactory(sslContext, hostnameVerifier);
+ return RegistryBuilder.<ConnectionSocketFactory> create().register("https", sslFactory).build();
+ }
+
+ private KeyStore loadPkcs(final String type, final String filename, final char[] password)
+ throws KeyStoreException, IOException, CertificateException, NoSuchAlgorithmException {
+ final KeyStore keyStore = KeyStore.getInstance(type);
+ final File key = ResourceUtils.getFile(filename);
+ try (InputStream in = new FileInputStream(key)) {
+ keyStore.load(in, password);
+ }
+ return keyStore;
+ }
+
+ /*
+ * Create a ClientConnectionPoolManager that maintains a pool of HttpClientConnections and is able to service connection
+ * requests from multiple execution threads. Connections are pooled on a per route basis. A request for a route which
+ * already the manager has persistent connections for available in the pool will be services by leasing a connection
+ * from the pool rather than creating a brand new connection.
+ */
+ private PoolingHttpClientConnectionManager buildConnectionManager(final HttpPoolConfiguration poolConfig,
+ final Registry<ConnectionSocketFactory> socketFactoryRegistry) {
+
+ final PoolingHttpClientConnectionManager connectionManager = (socketFactoryRegistry != null)
+ ? new PoolingHttpClientConnectionManager(socketFactoryRegistry)
+ : new PoolingHttpClientConnectionManager();
+
+ if (poolConfig != null) {
+ connectionManager.setMaxTotal(poolConfig.getMaxTotal());
+ // Default max per route is used in case it's not set for a specific route
+ connectionManager.setDefaultMaxPerRoute(poolConfig.getMaxPerRoute());
+
+ for (final HttpPoolConfiguration.HostConfiguration hostConfig : poolConfig.getHostConfigurations()) {
+ final HttpHost host = new HttpHost(hostConfig.getHost(), hostConfig.getPort(), hostConfig.getScheme());
+ // Max per route for a specific hosts route
+ connectionManager.setMaxPerRoute(new HttpRoute(host), hostConfig.getMaxPerRoute());
+ }
+ }
+ return connectionManager;
+ }
+
+ private RequestConfig buildRequestConfig() {
+ return RequestConfig.custom().setConnectionRequestTimeout(connectionRequestTimeout).setConnectTimeout(connectTimeout).setSocketTimeout(socketTimeout)
+ .build();
+ }
+
+ @Override
+ public RestTemplate getRestTemplate() {
+ return restTemplate;
+ }
+
+ @Override
+ public String getBaseUrl() {
+ return baseUrl;
+ }
+
+ public void setRestClientInterceptor(final List<ClientHttpRequestInterceptor> interceptors) {
+ restTemplate.setInterceptors(interceptors);
+ }
+
+}
diff --git a/deployment/roles/vitamui/templates/ingest-external/application.yml.j2 b/deployment/roles/vitamui/templates/ingest-external/application.yml.j2
index d06a3c5f4b75405f8727a2fd6c32c0553ac4d0eb..6b304c5786391963c36fef59b0eadfecd862b86a 100644
--- a/deployment/roles/vitamui/templates/ingest-external/application.yml.j2
+++ b/deployment/roles/vitamui/templates/ingest-external/application.yml.j2
@@ -9,10 +9,6 @@ spring:
tags: {{ consul_tags }}
instanceId: {{ vitamui_struct.vitamui_component }}-${server.port}-${spring.cloud.client.hostname}
-# should we fix some limit here ?
-spring.servlet.multipart.max-file-size: -1
-spring.servlet.multipart.max-request-size: -1
-
server-identity:
identityName: {{ vitamui_site_name }}
identityRole: vitamui-{{ vitamui_struct.vitamui_component }}
diff --git a/deployment/roles/vitamui/templates/ingest-internal/application.yml.j2 b/deployment/roles/vitamui/templates/ingest-internal/application.yml.j2
index be02b296aa2e90de70869c99f3c06d2b3f4bf98e..1dd28a2d123b9535e905e46c966197280843c4cf 100644
--- a/deployment/roles/vitamui/templates/ingest-internal/application.yml.j2
+++ b/deployment/roles/vitamui/templates/ingest-internal/application.yml.j2
@@ -11,10 +11,6 @@ spring:
mongodb:
uri: "mongodb://{{ mongodb.iam.user }}:{{ mongodb.iam.password }}@{{ mongodb.host }}:{{ mongodb.mongod_port }}/{{ mongodb.iam.db }}?replicaSet={{ mongod_replicaset_name }}&connectTimeoutMS={{ mongod_client_connect_timeout_ms }}"
-# should we fix some limit here ?
-spring.servlet.multipart.max-file-size: -1
-spring.servlet.multipart.max-request-size: -1
-
server-identity:
identityName: {{ vitamui_site_name }}
identityRole: vitamui-{{ vitamui_struct.vitamui_component }}
diff --git a/ui/ui-frontend/projects/ingest/src/app/core/common/ingest-list.ts b/ui/ui-frontend/projects/ingest/src/app/core/common/ingest-list.ts
index e84b4935dee976425361343055bb298c7b00c89b..080d7db4b9db95dc1fc051c343ac7f91b96d47a8 100644
--- a/ui/ui-frontend/projects/ingest/src/app/core/common/ingest-list.ts
+++ b/ui/ui-frontend/projects/ingest/src/app/core/common/ingest-list.ts
@@ -1,13 +1,15 @@
export enum IngestStatus {
- WIP, FINISHED, ERROR
+ WIP,
+ FINISHED,
+ ERROR,
}
export class IngestInfo {
- constructor(public name: string, public size: number, public nbChunks: number, public actualChunk: number, public status: IngestStatus) { }
+ constructor(public name: string, public size: number, public sizeUploaded: number, public status: IngestStatus) {}
}
export class IngestList {
- ingests: {[key: string]: IngestInfo} = {};
+ ingests: { [key: string]: IngestInfo } = {};
wipNumber = 0;
add(requestId: string, info: IngestInfo) {
@@ -15,18 +17,22 @@ export class IngestList {
this.wipNumber++;
}
- update(requestId: string, status?: IngestStatus) {
- if (!this.ingests[requestId]) { return; }
+ update(requestId: string, sizeUploaded: number, status?: IngestStatus) {
+ if (!this.ingests[requestId]) {
+ return;
+ }
- if (status) { // status defined and value > 0 ( FINISHED or ERROR )
+ if (status) {
+ // status defined and value > 0 ( FINISHED or ERROR )
this.ingests[requestId].status = status;
if (status === IngestStatus.FINISHED) {
this.finishTask(requestId);
}
}
- if (!status) { // status undefined or status = WIP (index = 0)
- this.ingests[requestId].actualChunk ++;
- if ( this.ingests[requestId].actualChunk === this.ingests[requestId].nbChunks ) {
+ if (!status) {
+ // status undefined or status = WIP (index = 0)
+ this.ingests[requestId].sizeUploaded = sizeUploaded;
+ if (this.ingests[requestId].sizeUploaded === this.ingests[requestId].size) {
this.finishTask(requestId);
}
}
@@ -34,6 +40,6 @@ export class IngestList {
finishTask(requestId: string) {
delete this.ingests[requestId];
- this.wipNumber --;
+ this.wipNumber--;
}
}
diff --git a/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.html b/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.html
index e848bda8f1674de0df8f51f59a5ad3ffa9a6d266..84322c27677ac946c7fcbdcf59ae84d963bfaf4c 100644
--- a/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.html
+++ b/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.html
@@ -2,28 +2,34 @@
<vitamui-common-progress-bar [index]="stepIndex" [count]="stepCount"></vitamui-common-progress-bar>
</div>
<form [formGroup]="sipForm">
- <div class="content">
- <ng-container [ngSwitch]="contextId">
- <ng-container *ngSwitchCase="'HOLDING_SCHEME'">
- <h4> {{'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_IMPORT_TYPE.HOLDING_SCHEME' | translate}}</h4>
- <h2>{{'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_LABEL_IMPORT_TYPE.HOLDING_SCHEME' | translate}}</h2>
- </ng-container>
- <ng-container *ngSwitchCase="'FILING_SCHEME'">
- <h4>{{'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_IMPORT_TYPE.FILING_SCHEME' | translate}}</h4>
- <h2>{{'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_LABEL_IMPORT_TYPE.FILING_SCHEME' | translate}}</h2>
+ <div class="content">
+ <ng-container [ngSwitch]="contextId">
+ <ng-container *ngSwitchCase="'HOLDING_SCHEME'">
+ <h4>{{ 'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_IMPORT_TYPE.HOLDING_SCHEME' | translate }}</h4>
+ <h2>{{ 'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_LABEL_IMPORT_TYPE.HOLDING_SCHEME' | translate }}</h2>
</ng-container>
- <ng-container *ngSwitchCase="'DEFAULT_WORKFLOW'">
- <h4>{{'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_IMPORT_TYPE.DEFAULT_WORKFLOW' | translate}}</h4>
- <h2>{{'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_LABEL_IMPORT_TYPE.DEFAULT_WORKFLOW' | translate}}</h2>
- </ng-container>¯
-
-</ng-container>
+ <ng-container *ngSwitchCase="'FILING_SCHEME'">
+ <h4>{{ 'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_IMPORT_TYPE.FILING_SCHEME' | translate }}</h4>
+ <h2>{{ 'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_LABEL_IMPORT_TYPE.FILING_SCHEME' | translate }}</h2>
+ </ng-container>
+ <ng-container *ngSwitchCase="'DEFAULT_WORKFLOW'">
+ <h4>{{ 'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_IMPORT_TYPE.DEFAULT_WORKFLOW' | translate }}</h4>
+ <h2>{{ 'INGEST_ACTION.MESSAGE_IDENTIFIER.MESSAGE_LABEL_IMPORT_TYPE.DEFAULT_WORKFLOW' | translate }}</h2> </ng-container
+ >¯
+ </ng-container>
<div class="d-flex">
- <div class="drag-and-drop-area" [ngClass]="{'on-over': hasDropZoneOver}" vitamuiCommonDragAndDrop
- (fileToUploadEmitter)="onDropped($event)" (fileDragOverEmitter)="onDragOver($event)" (fileDragLeaveEmitter)="onDragLeave($event)">
- <div *ngIf="fileName && fileSize>0 && fileSizeString" class="drag-container">
- <div class="file-info-class">{{ fileName }}
- <span class="text-grey" [ngStyle]="{'font-size':'13px'}"> | {{ fileSizeString }} </span>
+ <div
+ class="drag-and-drop-area"
+ [ngClass]="{ 'on-over': hasDropZoneOver }"
+ vitamuiCommonDragAndDrop
+ (fileToUploadEmitter)="onDropped($event)"
+ (fileDragOverEmitter)="onDragOver($event)"
+ (fileDragLeaveEmitter)="onDragLeave($event)"
+ >
+ <div *ngIf="fileName && fileSize > 0 && fileSizeString" class="drag-container">
+ <div class="file-info-class">
+ {{ fileName }}
+ <span class="text-grey" [ngStyle]="{ 'font-size': '13px' }"> | {{ fileSizeString }} </span>
<i class="material-icons success-icon" *ngIf="!hasError">check_circle</i>
<div>
<span class="text-red">{{ message }} </span>
@@ -35,23 +41,25 @@
<input type="file" #fileSearch class="input-file" (change)="handleFileInput($event.target.files)" />
<div class="drop-area">
- <div *ngIf="!fileSize || !hasSip" class="sip-drop">
- {{'INGEST_UPLOAD.ADD_FILE' | translate}} <br>
- </div>
+ <div *ngIf="!fileSize || !hasSip" class="sip-drop">{{ 'INGEST_UPLOAD.ADD_FILE' | translate }} <br /></div>
<div *ngIf="!fileSize || !hasSip">
<div class="sip-drop">
- <div class="upload"><span class="url-select" (click)="addSip()">{{'INGEST_UPLOAD.BROWSE' | translate}}</span>
+ <div class="upload">
+ <span class="url-select" (click)="addSip()">{{ 'INGEST_UPLOAD.BROWSE' | translate }}</span>
</div>
- <span class="sip-drop-small">{{'INGEST_UPLOAD.ADD_FILE_DESCRIPTION_1' | translate}}</span><br/>
- <span class="sip-drop-small">{{'INGEST_UPLOAD.ADD_FILE_DESCRIPTION_2' | translate}}</span>
+ <span class="sip-drop-small">{{ 'INGEST_UPLOAD.ADD_FILE_DESCRIPTION_1' | translate }}</span
+ ><br />
+ <span class="sip-drop-small">{{ 'INGEST_UPLOAD.ADD_FILE_DESCRIPTION_2' | translate }}</span>
</div>
</div>
</div>
</div>
</div>
<div class="actions">
- <button type="button" class="btn primary" [disabled]="isDisabled || hasError" (click)="upload()">{{'INGEST_UPLOAD.CONFIRME' | translate}}</button>
- <button type="button" class="btn cancel" (click)="onCancel()">{{'COMMON.CANCEL' | translate}}</button>
+ <button type="button" class="btn primary" [disabled]="isDisabled || hasError" (click)="upload()">
+ {{ 'INGEST_UPLOAD.CONFIRME' | translate }}
+ </button>
+ <button type="button" class="btn cancel" (click)="onCancel()">{{ 'COMMON.CANCEL' | translate }}</button>
</div>
</div>
</form>
diff --git a/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.ts b/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.ts
index ee2c01b65a67ec4b670fad09fe113b29e1c03de3..57149b6414de2b42b8bfdee64fe6cefb3fd4d667 100644
--- a/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.ts
+++ b/ui/ui-frontend/projects/ingest/src/app/core/common/upload.component.ts
@@ -34,27 +34,22 @@
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*/
-import { Component, OnInit, ViewChild, Inject } from '@angular/core';
+import { Component, Inject, OnInit, ViewChild } from '@angular/core';
import { FormBuilder, FormGroup } from '@angular/forms';
import { MatDialogRef, MAT_DIALOG_DATA } from '@angular/material/dialog';
import { MatSnackBar } from '@angular/material/snack-bar';
-
import { BytesPipe, Logger } from 'ui-frontend-common';
-import { UploadService } from './upload.service';
import { VitamUISnackBarComponent } from '../../shared/vitamui-snack-bar';
-
+import { UploadService } from './upload.service';
const LAST_STEP_INDEX = 2;
-const action = 'RESUME';
@Component({
selector: 'app-upload',
templateUrl: './upload.component.html',
- styleUrls: ['./upload.component.scss']
+ styleUrls: ['./upload.component.scss'],
})
-
export class UploadComponent implements OnInit {
-
sipForm: FormGroup;
hasSip: boolean;
hasDropZoneOver = false;
@@ -73,7 +68,6 @@ export class UploadComponent implements OnInit {
isDisabled = true;
public stepIndex = 0;
public stepCount = 2;
-
@ViewChild('fileSearch', { static: false }) fileSearch: any;
@@ -86,7 +80,7 @@ export class UploadComponent implements OnInit {
public logger: Logger
) {
this.sipForm = this.formBuilder.group({
- hasSip: null
+ hasSip: null,
});
this.tenantIdentifier = data.tenantIdentifier;
@@ -97,9 +91,7 @@ export class UploadComponent implements OnInit {
this.extensions = ['.zip', '.tar', '.tar.gz', '.tar.bz2'];
this.sipForm.get('hasSip').setValue(true);
this.hasSip = this.sipForm.get('hasSip').value;
-
}
-
onDragOver(inDropZone: boolean) {
this.hasDropZoneOver = inDropZone;
@@ -127,7 +119,7 @@ export class UploadComponent implements OnInit {
this.fileSizeString = transformer.transform(this.fileSize);
if (!this.checkFileExtension(this.fileName)) {
- this.message = 'Le fichier déposé n\'est pas au bon format';
+ this.message = "Le fichier déposé n'est pas au bon format";
this.hasError = true;
return;
}
@@ -143,34 +135,34 @@ export class UploadComponent implements OnInit {
}
upload() {
- if (!this.isValidSIP) { return; }
-
- this.uploadService.uploadFile(this.fileToUpload, this.contextId, action, this.tenantIdentifier)
- .subscribe(
- () => {
- this.dialogRef.close();
- this.displaySnackBar(true);
- },
- (error: any) => {
- console.error(error);
- this.message = error.message;
- });
+ if (!this.isValidSIP) {
+ return;
+ }
+
+ this.uploadService.uploadIngestV2(this.tenantIdentifier, this.fileToUpload, this.fileToUpload.name).subscribe(
+ () => {
+ this.dialogRef.close();
+ this.displaySnackBar(true);
+ },
+ (error: any) => {
+ console.error(error);
+ this.message = error.message;
+ }
+ );
}
displaySnackBar(uploadComplete: boolean) {
this.snackBar.openFromComponent(VitamUISnackBarComponent, {
panelClass: 'vitamui-snack-bar',
data: { type: 'fileUploaded', name: uploadComplete },
- duration: 10000
+ duration: 10000,
});
}
isValidSIP() {
- return this.sipForm.get('hasSip').value === false ||
- (this.sipForm.get('hasSip').value === true);
+ return this.sipForm.get('hasSip').value === false || this.sipForm.get('hasSip').value === true;
}
-
onCancel() {
this.dialogRef.close();
}
@@ -184,5 +176,4 @@ export class UploadComponent implements OnInit {
}
return false;
}
-
}
diff --git a/ui/ui-frontend/projects/ingest/src/app/core/common/upload.service.ts b/ui/ui-frontend/projects/ingest/src/app/core/common/upload.service.ts
index 615e8f47f5d1c55bb03c1e694d1407b71f4a28e4..6b10811082ba37c4f2c1e961a8aaca5d94af119b 100644
--- a/ui/ui-frontend/projects/ingest/src/app/core/common/upload.service.ts
+++ b/ui/ui-frontend/projects/ingest/src/app/core/common/upload.service.ts
@@ -34,30 +34,21 @@
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*/
+import { HttpClient, HttpEvent, HttpEventType, HttpHeaders, HttpRequest } from '@angular/common/http';
import { Injectable } from '@angular/core';
-import { HttpRequest, HttpHeaders, HttpResponse } from '@angular/common/http';
-import { IngestApiService } from '../api/ingest-api.service';
-import { retry } from 'rxjs/operators';
import { BehaviorSubject, Observable } from 'rxjs';
+import { IngestApiService } from '../api/ingest-api.service';
import { IngestInfo, IngestList, IngestStatus } from './ingest-list';
-const BYTES_PER_CHUNK = 1024 * 1024; // 1MB request
const tenantKey = 'X-Tenant-Id';
const contextIdKey = 'X-Context-Id';
const actionKey = 'X-Action';
-const chunkOffsetKey = 'X-Chunk-Offset';
-const totalSizeKey = 'X-Size-Total';
-const requestIdKey = 'X-Request-Id';
-
-const MAX_RETRIES = 3;
@Injectable()
export class UploadService {
-
uploadStatus = new BehaviorSubject<IngestList>(new IngestList());
- constructor( private ingestApiService: IngestApiService) {
- }
+ constructor(private ingestApiService: IngestApiService, private httpClient: HttpClient) {}
filesStatus(): BehaviorSubject<IngestList> {
return this.uploadStatus;
@@ -73,91 +64,57 @@ export class UploadService {
this.uploadStatus.next(map);
}
- updateFileStatus(requestId: string, status?: IngestStatus): void {
+ updateFileStatus(requestId: string, sizeUploaded: number, status?: IngestStatus): void {
const map: IngestList = this.uploadStatus.getValue();
- map.update(requestId, status);
+ map.update(requestId, sizeUploaded, status);
}
- uploadFile(file: File, contextId: string, action: string, tenantIdentifier: string): Observable<IngestList> {
- const totalSize = file.size;
- let start = 0;
- let end = (totalSize < BYTES_PER_CHUNK) ? totalSize : BYTES_PER_CHUNK;
- const nbChunks = Math.ceil(totalSize / BYTES_PER_CHUNK);
-
- const request = this.generateIngestRequest(tenantIdentifier, contextId, action, start, end, totalSize, file);
-
- this.ingestApiService.upload(request)
- .pipe( retry(MAX_RETRIES) )
- .subscribe(
- (event) => {
- if (event instanceof HttpResponse) {
- // We get the requestId with the first request.
- const requestId = event.headers.get(requestIdKey);
- this.addNewUploadFile(requestId, new IngestInfo(file.name, totalSize, nbChunks, 1, IngestStatus.WIP));
- console.log('First API Request Id : ' + requestId);
- start = end;
- end = start + BYTES_PER_CHUNK;
-
- if (start >= totalSize) {
- this.updateFileStatus(requestId, IngestStatus.FINISHED);
- return;
- }
-
- for (let pointer = start; pointer < totalSize; pointer += BYTES_PER_CHUNK) {
- this.uploadChunks(file, requestId, pointer, pointer + BYTES_PER_CHUNK, totalSize, tenantIdentifier, contextId, action);
- }
-
- }
- },
- (error) => {
- console.log(error);
- this.addNewUploadFile('error', new IngestInfo(file.name, totalSize, nbChunks, 1, IngestStatus.ERROR));
- }
- );
- return this.uploadStatus;
- }
-
- private uploadChunks(file: File, requestId: any, start: number, end: number, totalSize: any, tenantIdentifier: string,
- contextId: string, action: string ) {
-
- const request = this.generateIngestRequest(tenantIdentifier, contextId, action, start, end, totalSize, file, requestId);
- this.ingestApiService.upload(request).pipe(retry(MAX_RETRIES)).subscribe(
- (event) => {
- if (event instanceof HttpResponse) {
- this.updateFileStatus(requestId);
- }
- },
- (error) => {
- console.log(error);
- this.updateFileStatus(requestId, IngestStatus.ERROR);
- }
- );
- }
-
- private generateIngestRequest(
+ private uploadStreaming(
tenantIdentifier: string,
contextId: string,
action: string,
- start: number,
- end: number,
- totalSize: number,
- file: File,
- requestId?: string
- ): HttpRequest<FormData> {
+ file: Blob,
+ fileName: string
+ ): Observable<HttpEvent<void>> {
let headers = new HttpHeaders();
headers = headers.set(tenantKey, tenantIdentifier.toString());
- headers = headers.set(chunkOffsetKey, start.toString());
- headers = headers.set(totalSizeKey, totalSize.toString());
headers = headers.set(contextIdKey, contextId);
headers = headers.set(actionKey, action);
- if (requestId) {
- headers = headers.set(requestIdKey, requestId);
- }
-
- const formdata: FormData = new FormData();
- formdata.append('uploadedFile', file.slice(start, end), file.name);
-
- return new HttpRequest('POST', this.ingestApiService.getBaseUrl() + '/ingest/upload', formdata, { headers });
+ headers = headers.set('Content-Type', 'application/octet-stream');
+ headers = headers.set('reportProgress', 'true');
+ headers = headers.set('ngsw-bypass', 'true');
+ headers = headers.set('fileName', fileName);
+
+ const options = {
+ headers: headers,
+ responseType: 'text' as 'text',
+ reportProgress: true,
+ };
+ return this.httpClient.request(new HttpRequest('POST', this.ingestApiService.getBaseUrl() + '/ingest/upload-v2', file, options));
}
+ public uploadIngestV2(tenantIdentifier: string, file: Blob, fileName: string): Observable<IngestList> {
+ let progressPercent = 0;
+ this.addNewUploadFile(fileName, new IngestInfo(fileName, file.size, 0, IngestStatus.WIP));
+ this.uploadStreaming(tenantIdentifier, 'DEFAULT_WORKFLOW', 'RESUME', file, fileName).subscribe(
+ (data) => {
+ if (data) {
+ switch (data.type) {
+ case HttpEventType.UploadProgress:
+ progressPercent = Math.round((data.loaded / data.total) * 100);
+ this.updateFileStatus(fileName, progressPercent);
+ break;
+ case HttpEventType.Response:
+ this.updateFileStatus(fileName, 100, IngestStatus.FINISHED);
+ break;
+ }
+ }
+ },
+ (error) => {
+ this.updateFileStatus(fileName, IngestStatus.ERROR);
+ console.log('ERROR: ', error);
+ }
+ );
+ return this.uploadStatus;
+ }
}
diff --git a/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.html b/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.html
index 648c9d60fbe9c81671a8f38be695d7852acc6728..503bb0baaae6b44cbfd35d311717bfbe1cc60157 100644
--- a/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.html
+++ b/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.html
@@ -6,22 +6,34 @@
<mat-sidenav-content>
<div class="vitamui-heading">
<vitamui-common-title-breadcrumb>
- {{'INGEST_PAGE.TITLE' | translate}}
+ {{ 'INGEST_PAGE.TITLE' | translate }}
</vitamui-common-title-breadcrumb>
- <vitamui-common-banner [searchbarPlaceholder]="'INGEST_ACTION.SEARCH' | translate"
- i18n-placeholder="@@ingestSearchPlaceholder" (search)="onSearchSubmit($event)">
- <button class="btn secondary" (click)="refresh()" matTooltip="{{'INGEST_ACTION.ToolTip_Refresh' | translate}}"
- i18n-matTooltip="Proof safe info hint@@proofSafeInfo" matTooltipClass="vitamui-tooltip"
- [ngStyle]="{'margin-right' : '10px'}">
+ <vitamui-common-banner
+ [searchbarPlaceholder]="'INGEST_ACTION.SEARCH' | translate"
+ i18n-placeholder="@@ingestSearchPlaceholder"
+ (search)="onSearchSubmit($event)"
+ >
+ <button
+ class="btn secondary"
+ (click)="refresh()"
+ matTooltip="{{ 'INGEST_ACTION.ToolTip_Refresh' | translate }}"
+ i18n-matTooltip="Proof safe info hint@@proofSafeInfo"
+ matTooltipClass="vitamui-tooltip"
+ [ngStyle]="{ 'margin-right': '10px' }"
+ >
<i class="vitamui-icon vitamui-icon-refresh"></i>
- <span>{{'INGEST_ACTION.REFRESH' | translate}}</span>
+ <span>{{ 'INGEST_ACTION.REFRESH' | translate }}</span>
</button>
- <button class="btn primary" (click)="openImportSipDialog('DEFAULT_WORKFLOW')"
- matTooltip="{{'INGEST_ACTION.ToolTip_New_Ingest' | translate}}"
- i18n-matTooltip="Proof safe info hint@@proofSafeInfo" matTooltipClass="vitamui-tooltip">
+ <button
+ class="btn primary"
+ (click)="openImportSipDialog('DEFAULT_WORKFLOW')"
+ matTooltip="{{ 'INGEST_ACTION.ToolTip_New_Ingest' | translate }}"
+ i18n-matTooltip="Proof safe info hint@@proofSafeInfo"
+ matTooltipClass="vitamui-tooltip"
+ >
<i class="vitamui-icon vitamui-icon-archive-ingest"></i>
- <span>{{'INGEST_ACTION.NEW_INGEST' | translate}}</span>
+ <span>{{ 'INGEST_ACTION.NEW_INGEST' | translate }}</span>
</button>
</vitamui-common-banner>
@@ -29,26 +41,43 @@
<form [formGroup]="dateRangeFilterForm">
<div class="date-filter-container">
<div class="date-filter">
- <span *ngIf="!dateRangeFilterForm.get('startDate').value;else showStartDate" (click)="pickerStart.open()"
- i18n="@@apiSupervisionStartDate">{{'INGEST_ACTION.START_DATE' | translate}}</span>
+ <span
+ *ngIf="!dateRangeFilterForm.get('startDate').value; else showStartDate"
+ (click)="pickerStart.open()"
+ i18n="@@apiSupervisionStartDate"
+ >{{ 'INGEST_ACTION.START_DATE' | translate }}</span
+ >
<ng-template #showStartDate>
- <span (click)="pickerStart.open()">{{ dateRangeFilterForm.get('startDate').value |
- date:'dd/MM/yyyy' }}</span>
+ <span (click)="pickerStart.open()">{{ dateRangeFilterForm.get('startDate').value | date: 'dd/MM/yyyy' }}</span>
<i class="material-icons clear-date-icon clickable" (click)="clearDate('startDate')">clear</i>
</ng-template>
- <input class="hidden" size="0" [matDatepicker]="pickerStart" formControlName="startDate"
- [max]="dateRangeFilterForm.get('endDate').value">
+ <input
+ class="hidden"
+ size="0"
+ [matDatepicker]="pickerStart"
+ formControlName="startDate"
+ [max]="dateRangeFilterForm.get('endDate').value"
+ />
<mat-datepicker #pickerStart></mat-datepicker>
</div>
<div class="date-filter">
- <span *ngIf="!dateRangeFilterForm.get('endDate').value; else showEndDate" (click)="pickerEnd.open()"
- i18n="@@apiSupervisionEndDate">{{'INGEST_ACTION.END_DATE' | translate}}</span>
- <ng-template #showEndDate><span (click)="pickerEnd.open()">{{
- dateRangeFilterForm.get('endDate').value | date:'dd/MM/yyyy' }}
- </span> <i class="material-icons clear-date-icon clickable" (click)="clearDate('endDate')">clear</i>
+ <span
+ *ngIf="!dateRangeFilterForm.get('endDate').value; else showEndDate"
+ (click)="pickerEnd.open()"
+ i18n="@@apiSupervisionEndDate"
+ >{{ 'INGEST_ACTION.END_DATE' | translate }}</span
+ >
+ <ng-template #showEndDate
+ ><span (click)="pickerEnd.open()">{{ dateRangeFilterForm.get('endDate').value | date: 'dd/MM/yyyy' }} </span>
+ <i class="material-icons clear-date-icon clickable" (click)="clearDate('endDate')">clear</i>
</ng-template>
- <input class="hidden" size="0" [matDatepicker]="pickerEnd" formControlName="endDate"
- [min]="dateRangeFilterForm.get('startDate').value">
+ <input
+ class="hidden"
+ size="0"
+ [matDatepicker]="pickerEnd"
+ formControlName="endDate"
+ [min]="dateRangeFilterForm.get('startDate').value"
+ />
<mat-datepicker #pickerEnd></mat-datepicker>
</div>
</div>
@@ -57,11 +86,12 @@
</div>
<div class="vitamui-content">
<app-upload-tracking></app-upload-tracking>
- </div> <br>
+ </div>
+ <br />
<div class="vitamui-content">
- <h5 class="mt-0 mb-4">{{'INGEST_LIST.TABLE_NAME' | translate}}</h5>
+ <h5 class="mt-0 mb-4">{{ 'INGEST_LIST.TABLE_NAME' | translate }}</h5>
<app-ingest-list (ingestClick)="showIngest($event)" [search]="search" [filters]="filters"></app-ingest-list>
</div>
</mat-sidenav-content>
-</mat-sidenav-container>
\ No newline at end of file
+</mat-sidenav-container>
diff --git a/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.ts b/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.ts
index 665059054ebc8ef95ef4e35cb050eadf9485d355..8692288768a206de3418b6ea98039ce5f1f9dcdb 100644
--- a/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.ts
+++ b/ui/ui-frontend/projects/ingest/src/app/ingest/ingest.component.ts
@@ -34,26 +34,29 @@
* The fact that you are presently reading this means that you have had
* knowledge of the CeCILL-C license and that you accept its terms.
*/
-import { Component, OnInit, ViewChild, HostListener } from '@angular/core';
+import { Component, ElementRef, HostListener, OnInit, ViewChild } from '@angular/core';
import { FormBuilder, FormGroup } from '@angular/forms';
-import { ActivatedRoute, Router } from '@angular/router';
-
-import { IngestListComponent } from './ingest-list/ingest-list.component';
import { MatDialog, MatDialogConfig } from '@angular/material/dialog';
-
-import { GlobalEventService, SidenavPage, SearchBarComponent, AdminUserProfile, Direction } from 'ui-frontend-common';
+import { ActivatedRoute, Router } from '@angular/router';
+import { AdminUserProfile, Direction, GlobalEventService, SearchBarComponent, SidenavPage } from 'ui-frontend-common';
+import { IngestList } from '../core/common/ingest-list';
import { UploadComponent } from '../core/common/upload.component';
import { UploadService } from '../core/common/upload.service';
-import { IngestList } from '../core/common/ingest-list';
-
+import { IngestListComponent } from './ingest-list/ingest-list.component';
@Component({
selector: 'app-ingest',
templateUrl: './ingest.component.html',
- styleUrls: ['./ingest.component.scss']
+ styleUrls: ['./ingest.component.scss'],
})
export class IngestComponent extends SidenavPage<any> implements OnInit {
search: string;
+ progressPercent = 0;
+ uploadError = false;
+
+ uploadSucces = false;
+ uploadInProgress = false;
+
tenantIdentifier: string;
guard = true;
connectedUserInfo: AdminUserProfile;
@@ -62,23 +65,30 @@ export class IngestComponent extends SidenavPage<any> implements OnInit {
filters: any = {};
ingestList: IngestList = new IngestList();
- @ViewChild(SearchBarComponent, {static: true}) searchBar: SearchBarComponent;
- @ViewChild(IngestListComponent, {static: true}) ingestListComponent: IngestListComponent;
+ @ViewChild(SearchBarComponent, { static: true }) searchBar: SearchBarComponent;
+ @ViewChild(IngestListComponent, { static: true }) ingestListComponent: IngestListComponent;
+
+ @ViewChild('inputFile') inputFile: ElementRef;
- constructor( private router: Router, private route: ActivatedRoute,
- globalEventService: GlobalEventService, public dialog: MatDialog, private formBuilder: FormBuilder,
- private uploadSipService: UploadService) {
+ constructor(
+ private router: Router,
+ private route: ActivatedRoute,
+ globalEventService: GlobalEventService,
+ public dialog: MatDialog,
+ private formBuilder: FormBuilder,
+ private uploadSipService: UploadService
+ ) {
super(route, globalEventService);
- route.params.subscribe(params => {
+ route.params.subscribe((params) => {
this.tenantIdentifier = params.tenantIdentifier;
});
this.dateRangeFilterForm = this.formBuilder.group({
startDate: null,
- endDate: null
+ endDate: null,
});
- this.dateRangeFilterForm.controls.startDate.valueChanges.subscribe(value => {
+ this.dateRangeFilterForm.controls.startDate.valueChanges.subscribe((value) => {
this.filters.startDate = value;
this.ingestListComponent.filters = this.filters;
});
@@ -98,10 +108,10 @@ export class IngestComponent extends SidenavPage<any> implements OnInit {
clearDate(date: 'startDate' | 'endDate') {
if (date === 'startDate') {
- this.dateRangeFilterForm.get(date).reset(null, {emitEvent: false});
+ this.dateRangeFilterForm.get(date).reset(null, { emitEvent: false });
this.filters.startDate = null;
} else if (date === 'endDate') {
- this.dateRangeFilterForm.get(date).reset(null, {emitEvent: false});
+ this.dateRangeFilterForm.get(date).reset(null, { emitEvent: false });
this.filters.endDate = null;
} else {
console.error('clearDate() error: unknown date ' + date);
@@ -141,7 +151,7 @@ export class IngestComponent extends SidenavPage<any> implements OnInit {
dialogConfig.data = {
tenantIdentifier: this.tenantIdentifier,
- givenContextId: type
+ givenContextId: type,
};
const dialogRef = this.dialog.open(UploadComponent, dialogConfig);
@@ -153,7 +163,7 @@ export class IngestComponent extends SidenavPage<any> implements OnInit {
}
changeTenant(tenantIdentifier: number) {
- this.router.navigate(['..', tenantIdentifier], {relativeTo: this.route});
+ this.router.navigate(['..', tenantIdentifier], { relativeTo: this.route });
}
refresh() {
diff --git a/ui/ui-frontend/projects/ingest/src/app/ingest/upload-tracking/upload-tracking.component.html b/ui/ui-frontend/projects/ingest/src/app/ingest/upload-tracking/upload-tracking.component.html
index 8ee0c963e0c6995c1348ea54096d4fbe04c97446..abacfe60c775554df77b58394eb74a3d9d634e38 100644
--- a/ui/ui-frontend/projects/ingest/src/app/ingest/upload-tracking/upload-tracking.component.html
+++ b/ui/ui-frontend/projects/ingest/src/app/ingest/upload-tracking/upload-tracking.component.html
@@ -1,34 +1,35 @@
<div class="row">
<div class="col-11">
- <h5 class="mt-0 mb-4">{{'INGEST_TRACKING.TITLE' | translate}} <span>({{ ingestList.wipNumber }})</span></h5>
+ <h5 class="mt-0 mb-4">
+ {{ 'INGEST_TRACKING.TITLE' | translate }} <span>({{ ingestList.wipNumber }})</span>
+ </h5>
</div>
<div class="col-1">
<button class="btn-circle" (click)="toogleTracking()">
- <i class="material-icons" [@rotateAnimation]="displayTracking ? 'collapse' : 'expand'">keyboard_arrow_down</i>
+ <i class="material-icons" [@rotateAnimation]="displayTracking ? 'collapse' : 'expand'">keyboard_arrow_down</i>
</button>
</div>
</div>
<div *ngIf="displayTracking">
- <span *ngIf="ingestList.wipNumber === 0">{{'INGEST_TRACKING.MESSAGE' | translate}}</span>
+ <span *ngIf="ingestList.wipNumber === 0">{{ 'INGEST_TRACKING.MESSAGE' | translate }}</span>
<table class="vitamui-table" *ngIf="ingestList.wipNumber > 0">
<thead>
- <tr>
- <th>{{'INGEST_TRACKING.SIP_NAME' | translate}}</th>
- <th style="width: 100px">{{'INGEST_TRACKING.WEIGHT' | translate}}</th>
- <th>{{'INGEST_TRACKING.STATUS' | translate}}</th>
- </tr>
+ <tr>
+ <th>{{ 'INGEST_TRACKING.SIP_NAME' | translate }}</th>
+ <th style="width: 100px">{{ 'INGEST_TRACKING.WEIGHT' | translate }}</th>
+ <th>{{ 'INGEST_TRACKING.STATUS' | translate }}</th>
+ </tr>
</thead>
<tbody>
- <tr class="vitamui-table-row" *ngFor="let ingestInfo of ingestList.ingests | keyvalue">
- <td>{{ ingestInfo.value.name }}</td>
- <td>{{ ingestInfo.value.size | filesize }}</td>
- <td>
- <span class="purcent">{{ (ingestInfo.value.actualChunk * 100) / ingestInfo.value.nbChunks | number:'1.0-0' }} %</span>
- <mat-progress-bar mode="determinate" [value]="(ingestInfo.value.actualChunk * 100) / ingestInfo.value.nbChunks" class="stepper-progress-bar">
- </mat-progress-bar>
- </td>
- </tr>
+ <tr class="vitamui-table-row" *ngFor="let ingestInfo of ingestList.ingests | keyvalue">
+ <td>{{ ingestInfo.value.name }}</td>
+ <td>{{ ingestInfo.value.size | filesize }}</td>
+ <td>
+ <span class="purcent">{{ ingestInfo.value.sizeUploaded | number: '1.0-0' }} %</span>
+ <mat-progress-bar mode="determinate" [value]="ingestInfo.value.sizeUploaded" class="stepper-progress-bar"> </mat-progress-bar>
+ </td>
+ </tr>
</tbody>
</table>
</div>
diff --git a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/config/IngestContextConfiguration.java b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/config/IngestContextConfiguration.java
index becda33a68340dc1e9aad387875cfd0dbae7f6b0..4ea4837c5f726a0ae11e5554ac060b41c17bc8b1 100644
--- a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/config/IngestContextConfiguration.java
+++ b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/config/IngestContextConfiguration.java
@@ -44,6 +44,8 @@ import fr.gouv.vitamui.ingest.external.client.IngestExternalRestClient;
import fr.gouv.vitamui.ingest.external.client.IngestExternalRestClientFactory;
import fr.gouv.vitamui.ingest.external.client.IngestExternalWebClient;
import fr.gouv.vitamui.ingest.external.client.IngestExternalWebClientFactory;
+import fr.gouv.vitamui.ingest.external.client.IngestStreamingExternalRestClient;
+import fr.gouv.vitamui.ingest.external.client.IngestStreamingExternalRestClientFactory;
import fr.gouv.vitamui.ui.commons.property.UIProperties;
import fr.gouv.vitamui.ui.commons.security.SecurityConfig;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
@@ -69,11 +71,21 @@ public class IngestContextConfiguration extends AbstractContextConfiguration {
@Bean
@ConditionalOnMissingBean
@DependsOn("uiProperties")
- public IngestExternalRestClientFactory ingestExternalRestClientFactory(final IngestApplicationProperties uiProperties,
+ public IngestExternalRestClientFactory ingestExternalRestClientFactory(
+ final IngestApplicationProperties uiProperties,
RestTemplateBuilder restTemplateBuilder) {
return new IngestExternalRestClientFactory(uiProperties.getIngestExternalClient(), restTemplateBuilder);
}
+
+ @Bean
+ @ConditionalOnMissingBean
+ @DependsOn("uiProperties")
+ public IngestStreamingExternalRestClientFactory ingestStreamingExternalRestClientFactory(
+ final IngestApplicationProperties uiProperties) {
+ return new IngestStreamingExternalRestClientFactory(uiProperties.getIngestExternalClient());
+ }
+
@Bean
@ConditionalOnMissingBean
@DependsOn("uiProperties")
@@ -95,4 +107,12 @@ public class IngestContextConfiguration extends AbstractContextConfiguration {
return ingestExternalWebClientFactory.getIngestExternalWebClient();
}
+
+ @Bean
+ public IngestStreamingExternalRestClient ingestStreamingExternalRestClient(
+ final IngestStreamingExternalRestClientFactory ingestStreamingExternalRestClientFactory) {
+ return ingestStreamingExternalRestClientFactory.getIngestStreamingExternalRestClient();
+ }
+
+
}
diff --git a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/rest/IngestController.java b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/rest/IngestController.java
index 4ec69cdca8e172573c7c6af7bae61b0875f9871e..caed079fe5ea3cdd8796ed1b6f6a01afe8649d0d 100644
--- a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/rest/IngestController.java
+++ b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/rest/IngestController.java
@@ -53,12 +53,12 @@ import fr.gouv.vitamui.ingest.common.rest.RestApi;
import fr.gouv.vitamui.ingest.service.IngestService;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
+import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
-import org.springframework.util.StringUtils;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@@ -92,15 +92,15 @@ import java.util.concurrent.atomic.AtomicLong;
@Produces("application/json")
public class IngestController extends AbstractUiRestController {
- private final IngestService service;
+ private final IngestService ingestService;
private final Map<String, AtomicLong> uploadMap = new ConcurrentHashMap<>();
private static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestController.class);
@Autowired
- public IngestController(final IngestService service) {
- this.service = service;
+ public IngestController(final IngestService ingestService) {
+ this.ingestService = ingestService;
}
@ApiOperation(value = "Get entities paginated")
@@ -112,7 +112,7 @@ public class IngestController extends AbstractUiRestController {
@RequestParam final Optional<DirectionDto> direction) {
LOGGER.debug("getAllPaginated page={}, size={}, criteria={}, orderBy={}, ascendant={}", page, size, criteria,
orderBy, direction);
- return service.getAllPaginated(page, size, criteria, orderBy, direction, buildUiHttpContext());
+ return ingestService.getAllPaginated(page, size, criteria, orderBy, direction, buildUiHttpContext());
}
@ApiOperation(value = "Get one ingest operation details")
@@ -121,7 +121,7 @@ public class IngestController extends AbstractUiRestController {
public LogbookOperationDto getOne(final @PathVariable("id") String id) {
ParameterChecker.checkParameter("The Identifier is a mandatory parameter: ", id);
LOGGER.error("Get Ingest={}", id);
- return service.getOne(buildUiHttpContext(), id);
+ return ingestService.getOne(buildUiHttpContext(), id);
}
@ApiOperation(value = "download ODT Report for an ingest operation")
@@ -129,7 +129,7 @@ public class IngestController extends AbstractUiRestController {
public ResponseEntity<byte[]> generateODTReport(final @PathVariable("id") String id) {
ParameterChecker.checkParameter("The Identifier is a mandatory parameter: ", id);
LOGGER.debug("download ODT report for the ingest with id :{}", id);
- byte[] bytes = service.generateODTReport(buildUiHttpContext(), id).getBody();
+ byte[] bytes = ingestService.generateODTReport(buildUiHttpContext(), id).getBody();
return ResponseEntity.ok()
.contentType(MediaType.APPLICATION_OCTET_STREAM).header("Content-Disposition", "attachment")
.body(bytes);
@@ -138,71 +138,22 @@ public class IngestController extends AbstractUiRestController {
@ApiOperation(value = "Upload an SIP", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
@Consumes(MediaType.APPLICATION_OCTET_STREAM_VALUE)
- @PostMapping(CommonConstants.INGEST_UPLOAD)
+ @PostMapping(CommonConstants.INGEST_UPLOAD_V2)
public ResponseEntity<Void> ingest(
- @RequestHeader(value = CommonConstants.X_REQUEST_ID_HEADER) String requestId,
@RequestHeader(value = CommonConstants.X_TENANT_ID_HEADER) final String tenantId,
@RequestHeader(value = CommonConstants.X_ACTION) final String xAction,
@RequestHeader(value = CommonConstants.X_CONTEXT_ID) final String contextId,
- @RequestHeader(value = CommonConstants.X_CHUNK_OFFSET) final String chunkOffset,
- @RequestHeader(value = CommonConstants.X_SIZE_TOTAL) final String totalSize,
- @RequestParam(CommonConstants.MULTIPART_FILE_PARAM_NAME) final MultipartFile file)
- throws InvalidParseOperationException {
+ @RequestHeader(value = "fileName") final String fileName,
+ final InputStream inputStream) {
ParameterChecker
- .checkParameter("The requestId, tenantId, xAction and contextId are mandatory parameters : ", requestId,
+ .checkParameter("The tenantId, xAction and contextId are mandatory parameters : ",
tenantId, xAction, contextId);
- SafeFileChecker.checkSafeFilePath(file.getOriginalFilename());
- SanityChecker.checkParameter(requestId);
- LOGGER.debug("[{}] Upload File : {} - {} bytes", requestId, file.getOriginalFilename(), totalSize);
- if (StringUtils.isEmpty(requestId)) {
- throw new BadRequestException("Unable to start the upload of the file: request identifer is not set.");
- }
-
- if (!uploadMap.containsKey(requestId)) {
- uploadMap.put(requestId, new AtomicLong(0));
- }
-
- long writtenDataSize = 0;
- final long size = Long.parseLong(totalSize);
- final long offset = Long.parseLong(chunkOffset);
-
- InputStream in = null;
- Path tmpFilePath = Paths.get(System.getProperty(CommonConstants.VITAMUI_TEMP_DIRECTORY), requestId);
- FileChannel fileChannel = null;
- try (RandomAccessFile randomAccessFile = new RandomAccessFile(tmpFilePath.toString(), "rw")) {
- fileChannel = randomAccessFile.getChannel();
- final int writtenByte = fileChannel.write(ByteBuffer.wrap(file.getBytes()), offset);
- final AtomicLong writtenByteSize = uploadMap.get(requestId);
- writtenDataSize = writtenByteSize.addAndGet(writtenByte);
-
- if (writtenDataSize >= size) {
- fileChannel.force(false);
- uploadMap.remove(requestId);
- in = new FileInputStream(tmpFilePath.toFile());
- }
-
- LOGGER.debug("Total upload : {} {} ...", writtenDataSize, size);
- if (writtenDataSize >= size) {
- LOGGER.debug("Start uploading file ...");
- service.upload(buildUiHttpContext(), in, contextId, xAction, file.getOriginalFilename());
- }
- final HttpHeaders headers = new HttpHeaders();
- headers.add(CommonConstants.X_REQUEST_ID_HEADER, requestId);
- return new ResponseEntity<>(headers, HttpStatus.OK);
- } catch (final IOException exception) {
- try {
- LOGGER.info("Try to delete temp file {} ", tmpFilePath);
- Files.deleteIfExists(tmpFilePath);
- } catch (IOException e) {
- LOGGER.error("Error deleting temp file {} error {} ", tmpFilePath, e.getMessage());
- }
- final String message = String.format(
- "An error occurred during the upload [Request id: %s - ChunkOffset : %s - Total size : %s] : %s",
- requestId,
- chunkOffset, totalSize, exception.getMessage());
- throw new InternalServerException(message, exception);
-
- }
+ SafeFileChecker.checkSafeFilePath(fileName);
+ LOGGER.info("Start uploading file ...{} ", fileName);
+ ResponseEntity<Void> response =
+ ingestService.streamingUpload(buildUiHttpContext(), fileName, inputStream, contextId, xAction);
+ LOGGER.info("The response in ui Ingest is {} ", response.toString());
+ return new ResponseEntity<>(HttpStatus.OK);
}
}
diff --git a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/service/IngestService.java b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/service/IngestService.java
index 66a88def639727fe5f89a34cfba8499655f91d95..4fccfb0ee47a22d8a991e69c139e0cbbd7550719 100644
--- a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/service/IngestService.java
+++ b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/service/IngestService.java
@@ -44,7 +44,7 @@ import fr.gouv.vitamui.commons.rest.client.ExternalHttpContext;
import fr.gouv.vitamui.commons.vitam.api.dto.LogbookOperationDto;
import fr.gouv.vitamui.ingest.external.client.IngestExternalRestClient;
import fr.gouv.vitamui.ingest.external.client.IngestExternalWebClient;
-import fr.gouv.vitamui.ingest.thread.IngestThread;
+import fr.gouv.vitamui.ingest.external.client.IngestStreamingExternalRestClient;
import fr.gouv.vitamui.ui.commons.service.AbstractPaginateService;
import fr.gouv.vitamui.ui.commons.service.CommonService;
import org.springframework.beans.factory.annotation.Autowired;
@@ -64,24 +64,28 @@ public class IngestService extends AbstractPaginateService<LogbookOperationDto>
private final IngestExternalWebClient ingestExternalWebClient;
private final IngestExternalRestClient ingestExternalRestClient;
+ private final IngestStreamingExternalRestClient ingestStreamingExternalRestClient;
private CommonService commonService;
@Autowired
public IngestService(final CommonService commonService, final IngestExternalRestClient ingestExternalRestClient,
- final IngestExternalWebClient ingestExternalWebClient) {
+ final IngestExternalWebClient ingestExternalWebClient,
+ final IngestStreamingExternalRestClient ingestStreamingExternalRestClient) {
this.commonService = commonService;
this.ingestExternalRestClient = ingestExternalRestClient;
this.ingestExternalWebClient = ingestExternalWebClient;
+ this.ingestStreamingExternalRestClient = ingestStreamingExternalRestClient;
}
@Override
- public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(final Integer page, final Integer size, final Optional<String> criteria,
- final Optional<String> orderBy, final Optional<DirectionDto> direction, final ExternalHttpContext context) {
+ public PaginatedValuesDto<LogbookOperationDto> getAllPaginated(final Integer page, final Integer size,
+ final Optional<String> criteria,
+ final Optional<String> orderBy, final Optional<DirectionDto> direction, final ExternalHttpContext context) {
return super.getAllPaginated(page, size, criteria, orderBy, direction, context);
}
public LogbookOperationDto getOne(final ExternalHttpContext context, final String id) {
- return super.getOne(context,id);
+ return super.getOne(context, id);
}
@Override
@@ -89,17 +93,7 @@ public class IngestService extends AbstractPaginateService<LogbookOperationDto>
return commonService.checkPagination(page, size);
}
- public void upload(final ExternalHttpContext context, InputStream in, final String contextId, final String action,
- final String originalFilename) {
-
- final IngestThread
- ingestThread =
- new IngestThread(ingestExternalWebClient, context, in, contextId, action, originalFilename);
-
- ingestThread.start();
- }
-
- public ResponseEntity<byte[]> generateODTReport(ExternalHttpContext context, String id) {
+ public ResponseEntity<byte[]> generateODTReport(ExternalHttpContext context, String id) {
return ingestExternalRestClient.generateODTReport(context, id);
}
@@ -107,4 +101,12 @@ public class IngestService extends AbstractPaginateService<LogbookOperationDto>
return ingestExternalRestClient;
}
+ public ResponseEntity<Void> streamingUpload(final ExternalHttpContext context, String fileName,
+ InputStream inputStream,
+ final String contextId,
+ final String action) {
+ return ingestStreamingExternalRestClient.streamingUpload(context, fileName, inputStream, contextId,
+ action);
+ }
+
}
diff --git a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/thread/IngestThread.java b/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/thread/IngestThread.java
deleted file mode 100644
index fca5f55bdd8e4bcb1ce32b65c65870a32a1ebd46..0000000000000000000000000000000000000000
--- a/ui/ui-ingest/src/main/java/fr/gouv/vitamui/ingest/thread/IngestThread.java
+++ /dev/null
@@ -1,60 +0,0 @@
-package fr.gouv.vitamui.ingest.thread;
-
-import fr.gouv.vitamui.commons.api.logger.VitamUILogger;
-import fr.gouv.vitamui.commons.api.logger.VitamUILoggerFactory;
-import fr.gouv.vitamui.commons.rest.client.ExternalHttpContext;
-import fr.gouv.vitamui.ingest.external.client.IngestExternalWebClient;
-import fr.gouv.vitamui.ingest.service.IngestService;
-import org.springframework.web.reactive.function.client.ClientResponse;
-
-import java.io.InputStream;
-
-/**
- * Thread that send the uploaded stream to vitamui ingest-external through it's client
- */
-public class IngestThread extends Thread {
-
- static final VitamUILogger LOGGER = VitamUILoggerFactory.getInstance(IngestService.class);
-
- private final IngestExternalWebClient client;
-
- private final ExternalHttpContext context;
-
- private final String originalFilename;
-
- private final String contextId;
- private final String action;
- private final InputStream in;
-
- public IngestThread(final IngestExternalWebClient client, final ExternalHttpContext context, InputStream in,
- final String contextId, final String action, final String originalFilename) {
- this.client = client;
- this.originalFilename = originalFilename;
- this.context = context;
- this.contextId = contextId;
- this.action = action;
- this.in = in;
- }
-
-
- @Override
- public void run() {
- ClientResponse response = null;
- try {
- response = client.upload(context, in, contextId, action, originalFilename);
- if (!response.statusCode().is2xxSuccessful()) {
- LOGGER.debug("Upload of [{}] failed. StatusCode : [{}] .", originalFilename,
- response.statusCode());
- }
-
- if (response.statusCode().is2xxSuccessful()) {
- LOGGER.debug("Upload of [{}] succeeded with StatusCode : [{}].", originalFilename,
- response.statusCode());
- }
-
- } catch (final Exception e) {
- LOGGER.debug("ERROR : Upload of [{}] failed.\n [{}]", originalFilename, e.getMessage());
- }
- }
-
-}
diff --git a/ui/ui-ingest/src/test/java/fr/gouv/vitamui/ingest/service/IngestServiceTest.java b/ui/ui-ingest/src/test/java/fr/gouv/vitamui/ingest/service/IngestServiceTest.java
index cb1c1ea6d3059883b8791ba013f99b9b51b7d23d..71e93fff4a45e96a027440b890c053626a085bf8 100644
--- a/ui/ui-ingest/src/test/java/fr/gouv/vitamui/ingest/service/IngestServiceTest.java
+++ b/ui/ui-ingest/src/test/java/fr/gouv/vitamui/ingest/service/IngestServiceTest.java
@@ -2,6 +2,7 @@ package fr.gouv.vitamui.ingest.service;
import fr.gouv.vitamui.ingest.external.client.IngestExternalRestClient;
import fr.gouv.vitamui.ingest.external.client.IngestExternalWebClient;
+import fr.gouv.vitamui.ingest.external.client.IngestStreamingExternalRestClient;
import fr.gouv.vitamui.ui.commons.service.CommonService;
import org.junit.Before;
import org.junit.Test;
@@ -11,8 +12,6 @@ import org.springframework.test.context.junit4.SpringRunner;
/**
* Unit test for {@link IngestService}.
- *
- *
*/
@RunWith(SpringRunner.class)
public class IngestServiceTest {
@@ -21,16 +20,20 @@ public class IngestServiceTest {
@Mock
private IngestExternalRestClient ingestExternalRestClient;
+ @Mock
+ private IngestStreamingExternalRestClient ingestStreamingExternalRestClient;
@Mock
private IngestExternalWebClient ingestExternalWebClient;
+
@Mock
private CommonService commonService;
@Before
public void init() {
- ingestService = new IngestService(commonService, ingestExternalRestClient, ingestExternalWebClient);
+ ingestService = new IngestService(commonService, ingestExternalRestClient, ingestExternalWebClient,
+ ingestStreamingExternalRestClient);
}
@Test
diff --git a/ui/ui-ingest/src/test/resources/ui-ingest-application.yml b/ui/ui-ingest/src/test/resources/ui-ingest-application.yml
index 32cc10eb859c13e3efd2f9969958537c4a7a4a46..6f0296462f3b44b4f76b57a0ff776af894fdd420 100644
--- a/ui/ui-ingest/src/test/resources/ui-ingest-application.yml
+++ b/ui/ui-ingest/src/test/resources/ui-ingest-application.yml
@@ -43,7 +43,6 @@ ui-ingest:
key-path: src/main/config/truststore.jks
key-password: jkspasswd
hostname-verification: false
-
ui-prefix: ingest-api
server-identity: