View Javadoc
1   /*
2    * This file is part of Waarp Project (named also Waarp or GG).
3    *
4    *  Copyright (c) 2019, Waarp SAS, and individual contributors by the @author
5    *  tags. See the COPYRIGHT.txt in the distribution for a full listing of
6    * individual contributors.
7    *
8    *  All Waarp Project is free software: you can redistribute it and/or
9    * modify it under the terms of the GNU General Public License as published by
10   * the Free Software Foundation, either version 3 of the License, or (at your
11   * option) any later version.
12   *
13   * Waarp is distributed in the hope that it will be useful, but WITHOUT ANY
14   * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15   * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16   *
17   *  You should have received a copy of the GNU General Public License along with
18   * Waarp . If not, see <http://www.gnu.org/licenses/>.
19   */
20  
21  package org.waarp.vitam.ingest;
22  
23  import fr.gouv.vitam.common.exception.InvalidParseOperationException;
24  import fr.gouv.vitam.common.json.JsonHandler;
25  import fr.gouv.vitam.ingest.external.client.IngestExternalClient;
26  import fr.gouv.vitam.ingest.external.client.IngestExternalClientFactory;
27  import org.apache.commons.cli.CommandLine;
28  import org.apache.commons.cli.Option;
29  import org.waarp.common.guid.GUID;
30  import org.waarp.common.logging.SysErrLogger;
31  import org.waarp.common.logging.WaarpLogger;
32  import org.waarp.common.logging.WaarpLoggerFactory;
33  import org.waarp.common.utility.SystemPropertyUtil;
34  import org.waarp.vitam.common.waarp.ManagerToWaarp;
35  import org.waarp.vitam.common.waarp.ManagerToWaarpFactory;
36  
37  import java.io.File;
38  import java.io.FilenameFilter;
39  import java.io.IOException;
40  import java.nio.file.Files;
41  import java.util.ArrayList;
42  import java.util.List;
43  import java.util.Properties;
44  
45  /**
46   * Factory that handles IngestRequest within a directory
47   */
48  public class IngestRequestFactory {
49    public static final String DEFAULT_INGEST_FACTORY =
50        "/waarp/data/r66/IngestFactory";
51    static final String ORG_WAARP_INGEST_BASEDIR = "org.waarp.ingest.basedir";
52    /**
53     * Internal Logger
54     */
55    private static final WaarpLogger logger =
56        WaarpLoggerFactory.getLogger(IngestRequestFactory.class);
57    private static final String WORK = "work";
58    private static final String BASENAME =
59        IngestRequest.class.getSimpleName() + ".";
60    private static final String EXTENSION = ".json";
61    private static final String RESULT_EXTENSION = ".xml";
62    private static final FilenameFilter JSON_ONLY =
63        (dir, name) -> name.startsWith(BASENAME) && name.endsWith(EXTENSION);
64    private static final IngestRequestFactory FACTORY =
65        new IngestRequestFactory();
66    static boolean vitamTakeCareLocalFile = true;
67  
68    static {
69      setBaseDir(new File(DEFAULT_INGEST_FACTORY));
70    }
71  
72    private File baseDir;
73    private File workDir;
74    private IngestExternalClientFactory clientFactory =
75        IngestExternalClientFactory.getInstance();
76  
77    private IngestRequestFactory() {
78      // empty
79    }
80  
81    /**
82     * Common options
83     *
84     * @return common Options
85     */
86    static Option getDirectoryOption() {
87      Option property =
88          new Option("D", "Use value for property " + ORG_WAARP_INGEST_BASEDIR);
89      property.setArgName("property=value");
90      property.setArgs(2);
91      property.setValueSeparator('=');
92      return property;
93    }
94  
95    /**
96     * Common parse command line and setup Base Directory
97     *
98     * @param cmd
99     */
100   static void parseDirectoryOption(CommandLine cmd) {
101     if (cmd.hasOption('D')) {
102       Properties properties = cmd.getOptionProperties("D");
103       setBaseDir(new File(properties.getProperty(ORG_WAARP_INGEST_BASEDIR,
104                                                  DEFAULT_INGEST_FACTORY)));
105     } else if (SystemPropertyUtil.contains(ORG_WAARP_INGEST_BASEDIR)) {
106       setBaseDir(new File(SystemPropertyUtil.get(ORG_WAARP_INGEST_BASEDIR,
107                                                  DEFAULT_INGEST_FACTORY)));
108     } else {
109       setBaseDir(new File(DEFAULT_INGEST_FACTORY));
110     }
111   }
112 
113   /**
114    * Set Base Directory
115    *
116    * @param baseDirToSet
117    */
118   static void setBaseDir(File baseDirToSet) {
119     FACTORY.baseDir = baseDirToSet;
120     FACTORY.baseDir.mkdirs();
121     FACTORY.workDir = new File(FACTORY.baseDir, WORK);
122     FACTORY.workDir.mkdirs();
123   }
124 
125   /**
126    * @return the instance of the factory
127    */
128   public static IngestRequestFactory getInstance() {
129     return FACTORY;
130   }
131 
132   /**
133    * Used in JUnit
134    */
135   void setBaseDir() {
136     baseDir = FACTORY.baseDir;
137     workDir = FACTORY.workDir;
138   }
139 
140   /**
141    * @return the base directory
142    */
143   File getBaseDir() {
144     return baseDir;
145   }
146 
147   /**
148    * @return the Ingest Vitam client
149    */
150   public IngestExternalClient getClient() {
151     return clientFactory.getClient();
152   }
153 
154   /**
155    * @param ingestRequest
156    *
157    * @return the associated ManagerToWaarp
158    */
159   public ManagerToWaarp getManagerToWaarp(IngestRequest ingestRequest) {
160     return ManagerToWaarpFactory
161         .getManagerToWaarp(ingestRequest.getWaarpModel());
162   }
163 
164   /**
165    * Save the IngestRequest as a new one, creating file
166    *
167    * @param ingestRequest
168    *
169    * @throws InvalidParseOperationException
170    */
171   synchronized void saveNewIngestRequest(IngestRequest ingestRequest)
172       throws InvalidParseOperationException {
173     File newFile = new File(baseDir, getNewName());
174     ingestRequest.setJsonPath(newFile.getName());
175     JsonHandler.writeAsFile(ingestRequest, newFile);
176   }
177 
178   /**
179    * @return the unique name for JSON
180    */
181   private static String getNewName() {
182     synchronized (FACTORY) {
183       GUID guid = new GUID();
184       return BASENAME + guid.getId() + EXTENSION;
185     }
186   }
187 
188   /**
189    * Update the IngestRequest
190    *
191    * @param ingestRequest
192    *
193    * @return true if saved
194    *
195    * @throws InvalidParseOperationException
196    */
197   synchronized boolean saveIngestRequest(IngestRequest ingestRequest)
198       throws InvalidParseOperationException {
199     File existingFile = new File(baseDir, ingestRequest.getJsonPath());
200     if (existingFile.canRead()) {
201       JsonHandler.writeAsFile(ingestRequest, existingFile);
202       return true;
203     }
204     throw new InvalidParseOperationException("Json File does not exist");
205   }
206 
207   /**
208    * Clean and remove all traces of this IngestRequest
209    *
210    * @param ingestRequest
211    *
212    * @return true if totally done
213    */
214   synchronized boolean removeIngestRequest(IngestRequest ingestRequest) {
215     if (ingestRequest.getJsonPath() != null) {
216       File existingFile = new File(baseDir, ingestRequest.getJsonPath());
217       boolean status = deleteFile(existingFile);
218       // Vitam is supposed to take care of this
219       if (vitamTakeCareLocalFile) {
220         File sourceFile = new File(ingestRequest.getPath());
221         status &= deleteFile(sourceFile);
222       }
223       // Delete the ATR file if any
224       File xmlAtrFile = getXmlAtrFile(ingestRequest);
225       status &= deleteFile(xmlAtrFile);
226       IngestRequest.IngestStep.endSessionMachineSate(ingestRequest.step);
227       // Ensure file are deleted there
228       while (existingFile.exists()) {
229         try {
230           FACTORY.wait(10);
231         } catch (InterruptedException ignore) {//NOSONAR
232           logger.debug(ignore);
233         }
234       }
235       return status;
236     }
237     return false;
238   }
239 
240   /**
241    * Internal
242    *
243    * @param file
244    *
245    * @return true if done
246    */
247   private boolean deleteFile(File file) {
248     if (file.canRead()) {
249       try {
250         Files.delete(file.toPath());
251       } catch (IOException e) {
252         logger.warn("Cannot delete file", e);
253         return false;
254       }
255     }
256     return true;
257   }
258 
259   /**
260    * @param ingestRequest
261    *
262    * @return the File pointer to the XML ATR file
263    */
264   File getXmlAtrFile(IngestRequest ingestRequest) {
265     return new File(workDir, ingestRequest.getJsonPath() + RESULT_EXTENSION);
266   }
267 
268   /**
269    * @return the list of existing IngestRequests. Some can be not ready or ended
270    */
271   synchronized List<IngestRequest> getExistingIngests() {
272     List<IngestRequest> list = new ArrayList<>();
273     File[] files = baseDir.listFiles(JSON_ONLY);
274     if (files != null) {
275       for (File file : files) {
276         try {
277           IngestRequest ingestRequest =
278               JsonHandler.getFromFile(file, IngestRequest.class);
279           list.add(ingestRequest);
280         } catch (InvalidParseOperationException ignored) {
281           // File could be deleted during read operation
282           SysErrLogger.FAKE_LOGGER.ignoreLog(ignored);
283         }
284       }
285     }
286     return list;
287   }
288 
289   /**
290    * @param filename
291    *
292    * @return the IngestRequest if found
293    *
294    * @throws InvalidParseOperationException
295    */
296   synchronized IngestRequest getSpecificIngestRequest(String filename)
297       throws InvalidParseOperationException {
298     File file = new File(baseDir, filename);
299     if (file.exists()) {
300       return JsonHandler.getFromFile(file, IngestRequest.class);
301     }
302     throw new InvalidParseOperationException("Cannot find " + filename);
303   }
304 }