View Javadoc
1   /*
2    * This file is part of Waarp Project (named also Waarp or GG).
3    *
4    *  Copyright (c) 2019, Waarp SAS, and individual contributors by the @author
5    *  tags. See the COPYRIGHT.txt in the distribution for a full listing of
6    * individual contributors.
7    *
8    *  All Waarp Project is free software: you can redistribute it and/or
9    * modify it under the terms of the GNU General Public License as published by
10   * the Free Software Foundation, either version 3 of the License, or (at your
11   * option) any later version.
12   *
13   * Waarp is distributed in the hope that it will be useful, but WITHOUT ANY
14   * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
15   * A PARTICULAR PURPOSE. See the GNU General Public License for more details.
16   *
17   *  You should have received a copy of the GNU General Public License along with
18   * Waarp . If not, see <http://www.gnu.org/licenses/>.
19   */
20  package org.waarp.common.digest;
21  
22  import io.netty.buffer.ByteBuf;
23  
24  import java.io.File;
25  import java.io.FileInputStream;
26  import java.io.FileNotFoundException;
27  import java.io.IOException;
28  import java.io.InputStream;
29  import java.nio.ByteBuffer;
30  import java.nio.channels.FileChannel;
31  import java.nio.charset.Charset;
32  import java.security.MessageDigest;
33  import java.security.NoSuchAlgorithmException;
34  import java.util.Arrays;
35  import java.util.zip.Adler32;
36  import java.util.zip.CRC32;
37  import java.util.zip.Checksum;
38  
39  import static org.waarp.common.digest.WaarpBC.*;
40  
41  /**
42   * Class implementing digest like MD5, SHA1, SHAxxx.
43   * <br>
44   * <p>
45   * Some performance reports: (done using java -server option)
46   * <ul>
47   * <li>File based only:</li>
48   * <ul>
49   * <li>If ADLER32 is the referenced time ADLER32=1, CRC32=2.5, MD5=4, SHA1=7,
50   * SHA256=11, SHA512=25</li>
51   * </ul>
52   * <li>Buffer based only:</li>
53   * <ul>
54   * <li>JVM version is the fastest (+20%).</li>
55   * <li>If ADLER32 is the referenced time ADLER32=1, CRC32=2.5, MD5=4, SHA1=8,
56   * SHA256=13, SHA384=29,
57   * SHA512=31</li>
58   * </ul>
59   * </ul>
60   */
61  public class FilesystemBasedDigest {
62  
63    private static final String ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM =
64        " Algorithm not supported by this JVM";
65  
66    /**
67     * Format used for Files
68     */
69    public static final Charset UTF8 = Charset.forName("UTF-8");
70    protected static final byte[] EMPTY = {};
71    public static final int ZERO_COPY_CHUNK_SIZE = 64 * 1024;
72  
73    static {
74      initializedTlsContext();
75    }
76  
77    Checksum checksum;
78    MessageDigest digest;
79    DigestAlgo algo;
80  
81    /**
82     * Constructor of an independent Digest
83     *
84     * @param algo
85     *
86     * @throws NoSuchAlgorithmException
87     */
88    public FilesystemBasedDigest(final DigestAlgo algo)
89        throws NoSuchAlgorithmException {
90      initialize(algo);
91    }
92  
93    /**
94     * (Re)Initialize the digest
95     *
96     * @throws NoSuchAlgorithmException
97     */
98    public final void initialize() throws NoSuchAlgorithmException {
99      switch (algo) {
100       case ADLER32:
101         checksum = new Adler32();
102         return;
103       case CRC32:
104         checksum = new CRC32();
105         return;
106       case MD5:
107       case MD2:
108       case SHA1:
109       case SHA256:
110       case SHA384:
111       case SHA512:
112         final String algoname = algo.algoName;
113         try {
114           digest = MessageDigest.getInstance(algoname);
115         } catch (final NoSuchAlgorithmException e) {
116           throw new NoSuchAlgorithmException(
117               algo + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM, e);
118         }
119         return;
120       default:
121         throw new NoSuchAlgorithmException(
122             algo.algoName + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM);
123     }
124   }
125 
126   /**
127    * (Re)Initialize the digest
128    *
129    * @param algo
130    *
131    * @throws NoSuchAlgorithmException
132    */
133   public final void initialize(final DigestAlgo algo)
134       throws NoSuchAlgorithmException {
135     this.algo = algo;
136     initialize();
137   }
138 
139   public final DigestAlgo getAlgo() {
140     return algo;
141   }
142 
143   /**
144    * Update the digest with new bytes
145    *
146    * @param bytes
147    * @param offset
148    * @param length
149    */
150   public final void Update(final byte[] bytes, final int offset,
151                            final int length) {
152     switch (algo) {
153       case ADLER32:
154       case CRC32:
155         checksum.update(bytes, offset, length);
156         return;
157       case MD5:
158       case MD2:
159       case SHA1:
160       case SHA256:
161       case SHA384:
162       case SHA512:
163         digest.update(bytes, offset, length);
164     }
165   }
166 
167   private byte[] reusableBytes;
168 
169   /**
170    * One should call getOffset since offset within array could not 0
171    *
172    * @param buffer
173    *
174    * @return the array corresponding to this buffer (may be copied)
175    */
176   public final byte[] getBytes(final ByteBuf buffer) {
177     final byte[] bytes;
178     final int length = buffer.readableBytes();
179     if (buffer.hasArray()) {
180       bytes = buffer.array();
181     } else {
182       if (reusableBytes == null || reusableBytes.length != length) {
183         reusableBytes = new byte[length];
184       }
185       bytes = reusableBytes;
186       buffer.getBytes(buffer.readerIndex(), bytes, 0, length);
187     }
188     return bytes;
189   }
190 
191   /**
192    * @param buffer
193    *
194    * @return the offset for the getBytes array
195    */
196   public final int getOffset(final ByteBuf buffer) {
197     if (buffer.hasArray()) {
198       return buffer.arrayOffset();
199     }
200     return 0;
201   }
202 
203   /**
204    * Update the digest with new buffer
205    */
206   public final void Update(final ByteBuf buffer) {
207     final byte[] bytes = getBytes(buffer);
208     final int start = getOffset(buffer);
209     final int length = buffer.readableBytes();
210     Update(bytes, start, length);
211   }
212 
213   /**
214    * Update the digest with new buffer
215    */
216   public final void Update(final byte[] buffer) {
217     Update(buffer, 0, buffer.length);
218   }
219 
220   /**
221    * @return the digest in array of bytes
222    */
223   public final byte[] Final() {
224     switch (algo) {
225       case ADLER32:
226       case CRC32:
227         return Long.toOctalString(checksum.getValue()).getBytes(UTF8);
228       case MD5:
229       case MD2:
230       case SHA1:
231       case SHA256:
232       case SHA384:
233       case SHA512:
234         return digest.digest();
235     }
236     return EMPTY;
237   }
238 
239   /**
240    * All Algo that Digest Class could handle
241    */
242   public enum DigestAlgo {
243     CRC32("CRC32", 11), ADLER32("ADLER32", 9), MD5("MD5", 16), MD2("MD2", 16),
244     SHA1("SHA-1", 20), SHA256("SHA-256", 32), SHA384("SHA-384", 48),
245     SHA512("SHA-512", 64);
246 
247     public final String algoName;
248     public final int byteSize;
249 
250     /**
251      * @return the length in bytes of one Digest
252      */
253     public final int getByteSize() {
254       return byteSize;
255     }
256 
257     /**
258      * @return the length in Hex form of one Digest
259      */
260     public final int getHexSize() {
261       return byteSize * 2;
262     }
263 
264     DigestAlgo(final String algoName, final int byteSize) {
265       this.algoName = algoName;
266       this.byteSize = byteSize;
267     }
268 
269     public static DigestAlgo getFromName(final String name) {
270       try {
271         return valueOf(name);
272       } catch (final IllegalArgumentException ignore) {//NOSONAR
273         // ignore
274       }
275       if ("CRC32".equalsIgnoreCase(name)) {
276         return CRC32;
277       } else if ("ADLER32".equalsIgnoreCase(name)) {
278         return ADLER32;
279       } else if ("MD5".equalsIgnoreCase(name)) {
280         return MD5;
281       } else if ("MD2".equalsIgnoreCase(name)) {
282         return MD2;
283       } else if ("SHA-1".equalsIgnoreCase(name)) {
284         return SHA1;
285       } else if ("SHA-256".equalsIgnoreCase(name)) {
286         return SHA256;
287       } else if ("SHA-384".equalsIgnoreCase(name)) {
288         return SHA384;
289       } else if ("SHA-512".equalsIgnoreCase(name)) {
290         return SHA512;
291       } else {
292         throw new IllegalArgumentException("Digest Algo not found");
293       }
294     }
295   }
296 
297   /**
298    * Should a file MD5 be computed using FastMD5
299    */
300   private static boolean useFastMd5;
301 
302   /**
303    * @param dig1
304    * @param dig2
305    *
306    * @return True if the two digest are equals
307    */
308   public static boolean digestEquals(final byte[] dig1, final byte[] dig2) {
309     return MessageDigest.isEqual(dig1, dig2);
310   }
311 
312   /**
313    * @param dig1
314    * @param dig2
315    *
316    * @return True if the two digest are equals
317    */
318   public static boolean digestEquals(final String dig1, final byte[] dig2) {
319     final byte[] bdig1 = getFromHex(dig1);
320     return MessageDigest.isEqual(bdig1, dig2);
321   }
322 
323   /**
324    * get the byte array of the MD5 for the given FileInterface using Nio
325    * access
326    *
327    * @param f
328    *
329    * @return the byte array representing the MD5
330    *
331    * @throws IOException
332    */
333   public static byte[] getHashMd5Nio(final File f) throws IOException {
334     return getHash(f, true, DigestAlgo.MD5);
335   }
336 
337   /**
338    * get the byte array of the MD5 for the given FileInterface using standard
339    * access
340    *
341    * @param f
342    *
343    * @return the byte array representing the MD5
344    *
345    * @throws IOException
346    */
347   public static byte[] getHashMd5(final File f) throws IOException {
348     return getHash(f, false, DigestAlgo.MD5);
349   }
350 
351   /**
352    * Internal function for No NIO InputStream support
353    *
354    * @param in will be closed at the end of this call
355    * @param algo
356    * @param buf
357    *
358    * @return the digest
359    *
360    * @throws IOException
361    */
362   private static byte[] getHashNoNio(final InputStream in,
363                                      final DigestAlgo algo, final byte[] buf)
364       throws IOException {
365     // Not NIO
366     final Checksum checksum;
367     int size;
368     try {
369       switch (algo) {
370         case ADLER32:
371           checksum = new Adler32();
372           return getBytesCrc(in, buf, checksum);
373         case CRC32:
374           return getBytesCrc(in, buf, null);
375         case MD5:
376         case MD2:
377         case SHA1:
378         case SHA256:
379         case SHA384:
380         case SHA512:
381           final String algoname = algo.algoName;
382           final MessageDigest digest;
383           try {
384             digest = MessageDigest.getInstance(algoname);
385           } catch (final NoSuchAlgorithmException e) {
386             throw new IOException(algo + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM,
387                                   e);
388           }
389           while ((size = in.read(buf)) >= 0) {
390             digest.update(buf, 0, size);
391           }
392           return digest.digest();
393         default:
394           throw new IOException(
395               algo.algoName + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM);
396       }
397     } finally {
398       in.close();
399     }
400   }
401 
402   private static byte[] getBytesCrc(final InputStream in, final byte[] buf,
403                                     Checksum checksum) throws IOException {
404     int size;
405     if (checksum == null) { // not ADLER32
406       checksum = new CRC32();
407     }
408     while ((size = in.read(buf)) >= 0) {
409       checksum.update(buf, 0, size);
410     }
411     return Long.toOctalString(checksum.getValue()).getBytes(UTF8);
412   }
413 
414   /**
415    * Get the Digest for the file using the specified algorithm using access
416    * through NIO or not
417    *
418    * @param f
419    * @param nio
420    * @param algo
421    *
422    * @return the digest
423    *
424    * @throws IOException
425    */
426   public static byte[] getHash(final File f, final boolean nio,
427                                final DigestAlgo algo) throws IOException {
428     if (!f.exists()) {
429       throw new FileNotFoundException(f.toString());
430     }
431     FileInputStream in = null;
432     try {
433       long bufSize = f.length();
434       if (bufSize == 0) {
435         return EMPTY;
436       }
437       if (bufSize > ZERO_COPY_CHUNK_SIZE) {
438         bufSize = ZERO_COPY_CHUNK_SIZE;
439       }
440       byte[] buf = new byte[(int) bufSize];
441       in = new FileInputStream(f);
442       if (nio) { // NIO
443         final FileChannel fileChannel = in.getChannel();
444         try {
445           final ByteBuffer bb = ByteBuffer.wrap(buf);
446           final Checksum checksum;
447           int size;
448           switch (algo) {
449             case ADLER32:
450               checksum = new Adler32();
451               buf = getBytesCrcFileChannel(buf, fileChannel, bb, checksum);
452               break;
453             case CRC32:
454               buf = getBytesCrcFileChannel(buf, fileChannel, bb, null);
455               break;
456             case MD5:
457             case MD2:
458             case SHA1:
459             case SHA256:
460             case SHA384:
461             case SHA512:
462               final String algoname = algo.algoName;
463               final MessageDigest digest;
464               try {
465                 digest = MessageDigest.getInstance(algoname);
466               } catch (final NoSuchAlgorithmException e) {
467                 throw new IOException(
468                     algo + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM, e);
469               }
470               while ((size = fileChannel.read(bb)) >= 0) {
471                 digest.update(buf, 0, size);
472                 bb.clear();
473               }
474               buf = digest.digest();
475               break;
476             default:
477               throw new IOException(
478                   algo.algoName + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM);
479           }
480         } finally {
481           fileChannel.close();
482         }
483       } else { // Not NIO
484         return getHashNoNio(in, algo, buf);
485       }
486       return buf;
487     } finally {
488       if (in != null) {
489         try {
490           in.close();
491         } catch (final Exception ignored) {
492           // nothing
493         }
494       }
495     }
496   }
497 
498   private static byte[] getBytesCrcFileChannel(final byte[] buf,
499                                                final FileChannel fileChannel,
500                                                final ByteBuffer bb,
501                                                Checksum checksum)
502       throws IOException {
503     int size;
504     if (checksum == null) { // Not ADLER32
505       checksum = new CRC32();
506     }
507     while ((size = fileChannel.read(bb)) >= 0) {
508       checksum.update(buf, 0, size);
509       bb.clear();
510     }
511     return Long.toOctalString(checksum.getValue()).getBytes(UTF8);
512   }
513 
514   /**
515    * Get the Digest for the file using the specified algorithm using access
516    * through NIO or not
517    *
518    * @param stream will be closed at the end of this call
519    * @param algo
520    *
521    * @return the digest
522    *
523    * @throws IOException
524    */
525   public static byte[] getHash(final InputStream stream, final DigestAlgo algo)
526       throws IOException {
527     if (stream == null) {
528       throw new FileNotFoundException();
529     }
530     try {
531       final byte[] buf = new byte[ZERO_COPY_CHUNK_SIZE];
532       // Not NIO
533       return getHashNoNio(stream, algo, buf);
534     } catch (final IOException e) {
535       try {
536         stream.close();
537       } catch (final Exception ignored) {
538         // nothing
539       }
540       throw e;
541     }
542   }
543 
544   /**
545    * Get hash with given {@link ByteBuf} (from Netty)
546    *
547    * @param buffer this buffer will not be changed
548    * @param algo
549    *
550    * @return the hash
551    *
552    * @throws IOException
553    */
554   public static byte[] getHash(final ByteBuf buffer, final DigestAlgo algo)
555       throws IOException {
556     final Checksum checksum;
557     final byte[] bytes;
558     int start = 0;
559     final int length = buffer.readableBytes();
560     if (buffer.hasArray()) {
561       start = buffer.arrayOffset();
562       bytes = buffer.array();
563     } else {
564       bytes = new byte[length];
565       buffer.getBytes(buffer.readerIndex(), bytes);
566     }
567     switch (algo) {
568       case ADLER32:
569         checksum = new Adler32();
570         return getBytesCrcByteBuf(checksum, bytes, start, length);
571       case CRC32:
572         return getBytesCrcByteBuf(null, bytes, start, length);
573       case MD5:
574       case MD2:
575       case SHA1:
576       case SHA256:
577       case SHA384:
578       case SHA512:
579         return getBytesVarious(algo, bytes, start, length);
580       default:
581         throw new IOException(
582             algo.algoName + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM);
583     }
584   }
585 
586   /**
587    * Get hash with given byte array
588    *
589    * @param buffer this buffer will not be changed
590    * @param algo
591    *
592    * @return the hash
593    *
594    * @throws IOException
595    */
596   public static byte[] getHash(final byte[] buffer, final DigestAlgo algo)
597       throws IOException {
598     return getHash(buffer, buffer.length, algo);
599   }
600 
601   /**
602    * Get hash with given byte array
603    *
604    * @param buffer this buffer will not be changed
605    * @param length the real size of the buffer
606    * @param algo
607    *
608    * @return the hash
609    *
610    * @throws IOException
611    */
612   public static byte[] getHash(final byte[] buffer, final int length,
613                                final DigestAlgo algo) throws IOException {
614     switch (algo) {
615       case ADLER32:
616         return getBytesCrcByteBuf(new Adler32(), buffer, 0, length);
617       case CRC32:
618         return getBytesCrcByteBuf(null, buffer, 0, length);
619       case MD5:
620       case MD2:
621       case SHA1:
622       case SHA256:
623       case SHA384:
624       case SHA512:
625         return getBytesVarious(algo, buffer, 0, length);
626       default:
627         throw new IOException(
628             algo.algoName + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM);
629     }
630   }
631 
632   private static byte[] getBytesVarious(final DigestAlgo algo,
633                                         final byte[] bytes, final int start,
634                                         final int length) throws IOException {
635     final String algoname = algo.algoName;
636     final MessageDigest digest;
637     try {
638       digest = MessageDigest.getInstance(algoname);
639     } catch (final NoSuchAlgorithmException e) {
640       throw new IOException(algoname + ALGORITHM_NOT_SUPPORTED_BY_THIS_JVM, e);
641     }
642     digest.update(bytes, start, length);
643     return digest.digest();
644   }
645 
646   private static byte[] getBytesCrcByteBuf(Checksum checksum,
647                                            final byte[] bytes, final int start,
648                                            final int length) {
649     if (checksum == null) { // not ADLER32
650       checksum = new CRC32();
651     }
652     checksum.update(bytes, start, length);
653     return Long.toOctalString(checksum.getValue()).getBytes(UTF8);
654   }
655 
656   /**
657    * Get hash with given {@link ByteBuf} (from Netty)
658    *
659    * @param buffer ByteBuf to use to get the hash and this buffer will
660    *     not
661    *     be changed
662    *
663    * @return the hash
664    */
665   public static byte[] getHashMd5(final ByteBuf buffer) {
666     try {
667       return getHash(buffer, DigestAlgo.MD5);
668     } catch (final IOException e) {
669       throw new IllegalStateException(e);
670     }
671   }
672 
673   /**
674    * Internal representation of Hexadecimal Code
675    */
676   private static final char[] HEX_CHARS = {
677       '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'a', 'b', 'c', 'd', 'e',
678       'f',
679   };
680 
681   /**
682    * Get the hexadecimal representation as a String of the array of bytes
683    *
684    * @param hash
685    *
686    * @return the hexadecimal representation as a String of the array of bytes
687    */
688   public static String getHex(final byte[] hash) {
689     final char[] buf = new char[hash.length * 2];
690     for (int i = 0, x = 0; i < hash.length; i++) {
691       buf[x++] = HEX_CHARS[hash[i] >>> 4 & 0xf];
692       buf[x++] = HEX_CHARS[hash[i] & 0xf];
693     }
694     return new String(buf);
695   }
696 
697   /**
698    * Get the array of bytes representation of the hexadecimal String
699    *
700    * @param hex
701    *
702    * @return the array of bytes representation of the hexadecimal String
703    */
704   public static byte[] getFromHex(final String hex) {
705     final byte[] from = hex.getBytes(UTF8);
706     final byte[] hash = new byte[from.length / 2];
707     for (int i = 0, x = 0; i < hash.length; i++) {
708       byte code1 = from[x++];
709       byte code2 = from[x++];
710       if (code1 >= HEX_CHARS[10]) {
711         code1 -= HEX_CHARS[10] - 10;
712       } else {
713         code1 -= HEX_CHARS[0];
714       }
715       if (code2 >= HEX_CHARS[10]) {
716         code2 -= HEX_CHARS[10] - 10;
717       } else {
718         code2 -= HEX_CHARS[0];
719       }
720       hash[i] = (byte) ((code1 << 4) + (code2 & 0xFF));
721     }
722     return hash;
723   }
724 
725   private static final byte[] salt =
726       { 'G', 'o', 'l', 'd', 'e', 'n', 'G', 'a', 't', 'e' };
727 
728   /**
729    * Crypt a password
730    *
731    * @param pwd to crypt
732    *
733    * @return the crypted password
734    */
735   public static String passwdCrypt(final String pwd) {
736     final MessageDigest digest;
737     try {
738       digest = MessageDigest.getInstance(DigestAlgo.MD5.algoName);
739     } catch (final NoSuchAlgorithmException e) {
740       throw new IllegalStateException(e);
741     }
742     final byte[] bpwd = pwd.getBytes(UTF8);
743     for (int i = 0; i < 16; i++) {
744       digest.update(bpwd, 0, bpwd.length);
745       digest.update(salt, 0, salt.length);
746     }
747     return getHex(digest.digest());
748   }
749 
750   /**
751    * Crypt a password
752    *
753    * @param pwd to crypt
754    *
755    * @return the crypted password
756    */
757   public static byte[] passwdCrypt(final byte[] pwd) {
758     final MessageDigest digest;
759     try {
760       digest = MessageDigest.getInstance(DigestAlgo.MD5.algoName);
761     } catch (final NoSuchAlgorithmException e) {
762       throw new IllegalStateException(e);
763     }
764     for (int i = 0; i < 16; i++) {
765       digest.update(pwd, 0, pwd.length);
766       digest.update(salt, 0, salt.length);
767     }
768     return digest.digest();
769   }
770 
771   /**
772    * @param pwd
773    * @param cryptPwd
774    *
775    * @return True if the pwd is comparable with the cryptPwd
776    */
777   public static boolean equalPasswd(final String pwd, final String cryptPwd) {
778     final String asHex;
779     asHex = passwdCrypt(pwd);
780     return cryptPwd.equals(asHex);
781   }
782 
783   /**
784    * @param pwd
785    * @param cryptPwd
786    *
787    * @return True if the pwd is comparable with the cryptPwd
788    */
789   public static boolean equalPasswd(final byte[] pwd, final byte[] cryptPwd) {
790     final byte[] bytes;
791     bytes = passwdCrypt(pwd);
792     return Arrays.equals(cryptPwd, bytes);
793   }
794 
795 }