/*
    Conversion from UTF-8 to UTF-16.  Source code file.

	Stripped and modified for John the Ripper ; see ConvertUTF.[ch].original
	for the original content. magnum, 2009, 2010, 2011

	What was left in smbencrypt.c is now moved to these files too.
*/
/*
   (from smbencrypt.c)

   Unix SMB/Netbios implementation.
   Version 1.9.
   SMB parameters and setup
   Copyright (C) Andrew Tridgell 1992-1998
   Modified by Jeremy Allison 1995.
   (and hacked further by others)

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
/*
 * (from ConvertUTF.c)
 *
 * Copyright 2001-2004 Unicode, Inc.
 *
 * Disclaimer
 *
 * This source code is provided as is by Unicode, Inc. No claims are
 * made as to fitness for any particular purpose. No warranties of any
 * kind are expressed or implied. The recipient agrees to determine
 * applicability of information provided. If this file has been
 * purchased on magnetic or optical media from Unicode, Inc., the
 * sole remedy for any claim will be exchange of defective media
 * within 90 days of receipt.
 *
 * Limitations on Rights to Redistribute This Code
 *
 * Unicode, Inc. hereby grants the right to freely use the information
 * supplied in this file in the creation of products supporting the
 * Unicode Standard, and to make copies of this file in any form
 * for internal or external distribution as long as this notice
 * remains attached.
 */

/* ---------------------------------------------------------------------

    Author: Mark E. Davis, 1994.
    Rev History: Rick McGowan, fixes & updates May 2001.
    Sept 2001: fixed const & error conditions per
	mods suggested by S. Parent & A. Lillich.
    June 2002: Tim Dodd added detection and handling of incomplete
	source sequences, enhanced error detection, added casts
	to eliminate compiler warnings.
    July 2003: slight mods to back out aggressive FFFE detection.
    Jan 2004: updated switches in from-UTF8 conversions.
    Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions.

    See the header file "ConvertUTF.h" for complete documentation.


------------------------------------------------------------------------ */

#include "unicode.h"
#include "UnicodeData.h"
#include "encoding_data.h"

// here is the KOI8-r to Unicode conversion for KOI8-r characters from 0xA0 to 0xFF
static UTF16 KOI8_r_to_unicode_high128[] = {
0x2500,0x2502,0x250C,0x2510,0x2514,0x2518,0x251C,0x2524,0x252C,0x2534,0x253C,0x2580,0x2584,0x2588,0x258C,0x2590,
0x2591,0x2592,0x2593,0x2320,0x25A0,0x2219,0x221A,0x2248,0x2264,0x2265,0x00A0,0x2321,0x00B0,0x00B2,0x00B7,0x00F7,
0x2550,0x2551,0x2552,0x0451,0x2553,0x2554,0x2555,0x2556,0x2557,0x2558,0x2559,0x255A,0x255B,0x255C,0x255D,0x255E,
0x255F,0x2560,0x2561,0x0401,0x2562,0x2563,0x2564,0x2565,0x2566,0x2567,0x2568,0x2569,0x256A,0x256B,0x256C,0x00A9,
0x044E,0x0430,0x0431,0x0446,0x0434,0x0435,0x0444,0x0433,0x0445,0x0438,0x0439,0x043A,0x043B,0x043C,0x043D,0x043E,
0x043F,0x044F,0x0440,0x0441,0x0442,0x0443,0x0436,0x0432,0x044C,0x044B,0x0437,0x0448,0x044D,0x0449,0x0447,0x044A,
0x042E,0x0410,0x0411,0x0426,0x0414,0x0415,0x0424,0x0413,0x0425,0x0418,0x0419,0x041A,0x041B,0x041C,0x041D,0x041E,
0x041F,0x042F,0x0420,0x0421,0x0422,0x0423,0x0416,0x0412,0x042C,0x042B,0x0417,0x0428,0x042D,0x0429,0x0427,0x042A };

// here is the CP1251 to Unicode conversion for CP1251 characters from 0xA0 to 0xFF NOTE, there is no 0x98, but we have placed 0x98 there as a 'placeholder'
static UTF16 CP1251_to_unicode_high128[] = {
0x0402,0x0403,0x201A,0x0453,0x201E,0x2026,0x2020,0x2021,0x20AC,0x2030,0x0409,0x2039,0x040A,0x040C,0x040B,0x040F,
0x0452,0x2018,0x2019,0x201C,0x201D,0x2022,0x2013,0x2014,0x0098,0x2122,0x0459,0x203A,0x045A,0x045C,0x045B,0x045F,
0x00A0,0x040E,0x045E,0x0408,0x00A4,0x0490,0x00A6,0x00A7,0x0401,0x00A9,0x0404,0x00AB,0x00AC,0x00AD,0x00AE,0x0407,
0x00B0,0x00B1,0x0406,0x0456,0x0491,0x00B5,0x00B6,0x00B7,0x0451,0x2116,0x0454,0x00BB,0x0458,0x0405,0x0455,0x0457,
0x0410,0x0411,0x0412,0x0413,0x0414,0x0415,0x0416,0x0417,0x0418,0x0419,0x041A,0x041B,0x041C,0x041D,0x041E,0x041F,
0x0420,0x0421,0x0422,0x0423,0x0424,0x0425,0x0426,0x0427,0x0428,0x0429,0x042A,0x042B,0x042C,0x042D,0x042E,0x042F,
0x0430,0x0431,0x0432,0x0433,0x0434,0x0435,0x0436,0x0437,0x0438,0x0439,0x043A,0x043B,0x043C,0x043D,0x043E,0x043F,
0x0440,0x0441,0x0442,0x0443,0x0444,0x0445,0x0446,0x0447,0x0448,0x0449,0x044A,0x044B,0x044C,0x044D,0x044E,0x044F };

UTF16 utc2_upcase[0x10000]; /* used by md5_gen_fmt.c, hence not static */
UTF16 utc2_downcase[0x10000]; /* declared in unicode.h non-static, yet could be static otherwise */

/*
 * These are used to convert from an encoding, into unicode.  iso-8859-1 is trivial
 * (it is a simple direct 1 to 1 conversion.  however, the others are not quite
 * that trivial. Thus to keep the code 'simple', we simply use an array lookup
 * for ALL encoding translations.
 * There is a 'to' unicode
 *          a 'from' unicode
 *          a toUP  and a todown array
 * These array values are properly filled in at init() once we know the proper code page the user is using.
 */
static UTF16 CP_to_Unicode[0x100];
static UTF8  CP_from_Unicode[0x10000];
static UTF8  CP_up[0x100];
static UTF8  CP_down[0x100];

#include "md4.h"
#if !defined(uint16) && !defined(HAVE_UINT16_FROM_RPC_RPC_H)
#if (SIZEOF_SHORT == 4)
#define uint16 __ERROR___CANNOT_DETERMINE_TYPE_FOR_INT16;
#else /* SIZEOF_SHORT != 4 */
#define uint16 unsigned short
#endif /* SIZEOF_SHORT != 4 */
#endif

#if !defined(int16) && !defined(HAVE_INT16_FROM_RPC_RPC_H)
#if (SIZEOF_SHORT == 4)
#define int16 __ERROR___CANNOT_DETERMINE_TYPE_FOR_INT16;
#else /* SIZEOF_SHORT != 4 */
#define int16 short
#endif /* SIZEOF_SHORT != 4 */
#endif

#include <string.h>
#include "common.h"
#include "arch.h"
#include "byteorder.h"
#if !defined (NOT_JOHN)
#include "options.h"
#else
struct opts { int flags; };
#define FLG_UTF8 1
struct opts options;
#endif

#if ARCH_LITTLE_ENDIAN
#define BE_FIX(a) a
#else
#define BE_FIX(a) ( ((a&0xFF00)>>8) | ((a&0xFF)<<8) )
#endif

//Init values for NT hashing
#define INIT_A 0x67452301
#define INIT_B 0xefcdab89
#define INIT_C 0x98badcfe
#define INIT_D 0x10325476

#define SQRT_2 0x5a827999
#define SQRT_3 0x6ed9eba1

/*
 * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed
 * into the first byte, depending on how many bytes follow.  There are
 * as many entries in this table as there are UTF-8 sequence types.
 * (I.e., one byte sequence, two byte... etc.). Remember that sequencs
 * for *legal* UTF-8 will be 4 or fewer bytes total.
 */
static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC };

/*
 * Magic values subtracted from a buffer value during UTF8 conversion.
 * This table contains as many values as there might be trailing bytes
 * in a UTF-8 sequence. (Cut-down version, 4 and 5 are illegal).
 */
const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
		     0x03C82080UL, 0xFA082080UL, 0x82082080UL };

/*
 * Index into the table below with the first byte of a UTF-8 sequence to
 * get the number of trailing bytes that are supposed to follow it.
 * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is
 * left as-is for anyone who may want to do such conversion, which was
 * allowed in earlier algorithms.
 *
 * Cut-down version for speed. Use with [c & 0x3f]
 */
const char opt_trailingBytesUTF8[64] = {
	1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
	2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5
};

static const int halfShift  = 10; /* used for shifting by 10 bits */

static const UTF32 halfBase = 0x0010000UL;
static const UTF32 halfMask = 0x3FFUL;

#define UNI_SUR_HIGH_START  (UTF32)0xD800
#define UNI_SUR_HIGH_END    (UTF32)0xDBFF
#define UNI_SUR_LOW_START   (UTF32)0xDC00
#define UNI_SUR_LOW_END     (UTF32)0xDFFF

// Convert UTF-8 to NT UNICODE (UTF-16LE) string. Note that regardless of
// processor type this must be in intel (little-endian) format.
// This code is optimised for speed. Errors result in truncation.
//
// Normally returns number of UTF16 characters converted. If truncated,
// the number of UTF8 characters that was successfully read is returned
// instead, so we can truncate our saved_plain or whatever accordingly.
//
// The original function in ConvertUTF.c is called ConvertUTF8toUTF16()
inline int utf8towcs(UTF16 * target, unsigned int len, const UTF8 * source,
                     unsigned int sourceLen)
{
	const UTF16 *targetStart = target;
	const UTF16 *targetEnd = target + len;
	const UTF8 *sourceStart = source;
	const UTF8 *sourceEnd = source + sourceLen;
	UTF32 ch;
	unsigned int extraBytesToRead;

	while (source < sourceEnd) {
		if (*source < 0xC0) {
#if ARCH_LITTLE_ENDIAN
			*target++ = (UTF16) * source++;
#else
			UTF8 val = *source++;
			SSVAL(target, 0, val);
			++target;
#endif
			if (*source == 0)
				break;
			if (target >= targetEnd) {
				*target = 0;
				return -1 * (source - sourceStart);
			}
			continue;
		}
		ch = *source;
		// The original code in ConvertUTF.c has a much larger (slower)
		// lookup table including zeros. This point must not be reached
		// with *source < 0xC0
		extraBytesToRead =
		    opt_trailingBytesUTF8[ch & 0x3f];
		if (source + extraBytesToRead >= sourceEnd) {
			*target = 0;
			return -1 * (source - sourceStart);
		}
		switch (extraBytesToRead) {
		case 3:
			ch <<= 6;
			ch += *++source;
		case 2:
			ch <<= 6;
			ch += *++source;
		case 1:
			ch <<= 6;
			ch += *++source;
		case 0:
			++source;
			break;
		default:
			*target = 0;
			return -1 * (source - sourceStart);
		}
		ch -= offsetsFromUTF8[extraBytesToRead];
#if 0 /* This only supports UCS-2 */
#if ARCH_LITTLE_ENDIAN
		*target++ = (UTF16) ch;
#else
		SSVAL(target, 0, ch);
		++target;
#endif
#else /* This supports full UTF-16 with surrogate pairs */
		if (ch <= UNI_MAX_BMP) {  /* Target is a character <= 0xFFFF */
#if ARCH_LITTLE_ENDIAN
			*target++ = (UTF16) ch;
#else
			SSVAL(target, 0, ch);
			++target;
#endif
		} else {  /* target is a character in range 0xFFFF - 0x10FFFF. */
			if (target + 1 >= targetEnd) {
				source -= (extraBytesToRead+1); /* Back up source pointer! */
				*target = 0;
				return -1 * (source - sourceStart);
			}
			ch -= halfBase;
#if ARCH_LITTLE_ENDIAN
			*target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START);
			*target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START);
#else
			SSVAL(target, 0, (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START));
			++target;
			SSVAL(target, 0, (UTF16)((ch & halfMask) + UNI_SUR_LOW_START));
			++target;
#endif
		}
#endif
		if (*source == 0)
			break;
		if (target >= targetEnd) {
			*target = 0;
			return -1 * (source - sourceStart);
		}
	}
	*target = 0;		// Null-terminate
	return (target - targetStart);
}

// Convert a string into an NT UNICODE string. Note that regardless of
// processor type this must be in intel (little-endian) format.
//
// This version converts from UTF-8 if the --encoding=utf8 option was given to John
// and from the other character sets otherwise which is faster, since it is a simple
// table lookup, vs computing wide characters.
//
//NOTE, Solaris did not build properly with this function inlined. Thus
// the inline was removed (for now).
//inline
int plaintowcs(UTF16 * dst, unsigned int maxdstlen, const UTF8 * src,
                      unsigned int srclen)
{
	if (!(options.utf8)) {
		int i, trunclen = (int)srclen;
		if (trunclen > maxdstlen)
			trunclen = maxdstlen;

		for (i = 0; i < trunclen; i++) {
#if ARCH_LITTLE_ENDIAN
			*dst++ = CP_to_Unicode[*src++];
#else
			UTF16 val = CP_to_Unicode[*src++];
			SSVAL(dst, 0, val);
			++dst;
#endif
		}
		*dst = 0;
		if (i < srclen)
			return -i;
		else
			return i;
	} else {		// Convert from UTF-8
		return utf8towcs(dst, maxdstlen, src, srclen);
	}
}

// strlen of UTF16 (in characters, not bytes)
inline unsigned int strlen16(const UTF16 * str)
{
	unsigned int len = 0;
	while (*str++ != 0)
		len++;
	return len;
}

/*
 * Creates an MD4 Hash of the user's password in NT UNICODE.
 * This version honours the --encoding=utf8 flag and makes a couple
 * of formats utf8-aware with few further modifications.
 *
 * Now using Alain's fast NTLM up to 27 characters of plaintext (54 bytes
 * of UTF-16) and a generic one (Solar's MD4) otherwise.
 *
 * MD4 compress function
 * Written by Alain Espinosa <alainesp@gmail.com> in 2008
 * and placed in the public domain.
 *
 */
int E_md4hash(const UTF8 * passwd, unsigned int len, unsigned char *p16)
{
	int trunclen;
	static UTF16 wpwd[PLAINTEXT_BUFFER_SIZE + 1];

	// Quick work around. We may want to port the else{} into BE code,
	// and do it without impacting LE speed. The boost is not huge though.
#if ARCH_ALLOWS_UNALIGNED
	if (len > 27) {
#endif
		static MD4_CTX ctx;

		/* Password is converted to UTF-16LE */
		trunclen = plaintowcs(wpwd, PLAINTEXT_BUFFER_SIZE, passwd, len);
		if(trunclen < 0)
			len = strlen16(wpwd); // From UTF-8 you can't know
		else
			len = trunclen;

		MD4_Init(&ctx);
		MD4_Update(&ctx, (unsigned char *)wpwd, len * sizeof(UTF16));
		MD4_Final(p16, &ctx);
#if ARCH_ALLOWS_UNALIGNED
	} else {
		static unsigned int nt_buffer[16];
		static unsigned int lastlen = 16;
		unsigned int a = INIT_A;
		unsigned int b = INIT_B;
		unsigned int c = INIT_C;
		unsigned int d = INIT_D;
		unsigned int i = 0, md4_size = 0;

		/* Password is converted to UTF-16LE */
		trunclen = plaintowcs(wpwd, 27, passwd, len);
		// We need to check this because it's not just a matter of truncating
		// length, it can be malformed UTF-8
		if (trunclen < 0)
			len = strlen16(wpwd);	/* From UTF-8 you can't know */
		else
			len = trunclen;

		/* Key setup */
		for (; md4_size + 1 < len; i++, md4_size += 2)
			nt_buffer[i] = wpwd[md4_size] | (wpwd[md4_size + 1] << 16);
		if (md4_size < len)
			nt_buffer[i] = wpwd[md4_size++] | 0x800000;
		else
			nt_buffer[i] = 0x80;

		/* Key cleaning */
		for (i++; i <= lastlen; i++)
			nt_buffer[i] = 0;

		lastlen = md4_size >> 1;
		nt_buffer[14] = md4_size << 4;

		/* Round 1 */
		a += (d ^ (b & (c ^ d)))  +  nt_buffer[0]  ;a = (a << 3 ) | (a >> 29);
		d += (c ^ (a & (b ^ c)))  +  nt_buffer[1]  ;d = (d << 7 ) | (d >> 25);
		c += (b ^ (d & (a ^ b)))  +  nt_buffer[2]  ;c = (c << 11) | (c >> 21);
		b += (a ^ (c & (d ^ a)))  +  nt_buffer[3]  ;b = (b << 19) | (b >> 13);

		a += (d ^ (b & (c ^ d)))  +  nt_buffer[4]  ;a = (a << 3 ) | (a >> 29);
		d += (c ^ (a & (b ^ c)))  +  nt_buffer[5]  ;d = (d << 7 ) | (d >> 25);
		c += (b ^ (d & (a ^ b)))  +  nt_buffer[6]  ;c = (c << 11) | (c >> 21);
		b += (a ^ (c & (d ^ a)))  +  nt_buffer[7]  ;b = (b << 19) | (b >> 13);

		a += (d ^ (b & (c ^ d)))  +  nt_buffer[8]  ;a = (a << 3 ) | (a >> 29);
		d += (c ^ (a & (b ^ c)))  +  nt_buffer[9]  ;d = (d << 7 ) | (d >> 25);
		c += (b ^ (d & (a ^ b)))  +  nt_buffer[10] ;c = (c << 11) | (c >> 21);
		b += (a ^ (c & (d ^ a)))  +  nt_buffer[11] ;b = (b << 19) | (b >> 13);

		a += (d ^ (b & (c ^ d)))  +  nt_buffer[12] ;a = (a << 3 ) | (a >> 29);
		d += (c ^ (a & (b ^ c)))  +  nt_buffer[13] ;d = (d << 7 ) | (d >> 25);
		c += (b ^ (d & (a ^ b)))  +  nt_buffer[14] ;c = (c << 11) | (c >> 21);
		b += (a ^ (c & (d ^ a))) ;b = (b << 19) | (b >> 13);

		/* Round 2 */
		a += ((b & (c | d)) | (c & d)) + nt_buffer[0] +SQRT_2; a = (a<<3 ) | (a>>29);
		d += ((a & (b | c)) | (b & c)) + nt_buffer[4] +SQRT_2; d = (d<<5 ) | (d>>27);
		c += ((d & (a | b)) | (a & b)) + nt_buffer[8] +SQRT_2; c = (c<<9 ) | (c>>23);
		b += ((c & (d | a)) | (d & a)) + nt_buffer[12]+SQRT_2; b = (b<<13) | (b>>19);

		a += ((b & (c | d)) | (c & d)) + nt_buffer[1] +SQRT_2; a = (a<<3 ) | (a>>29);
		d += ((a & (b | c)) | (b & c)) + nt_buffer[5] +SQRT_2; d = (d<<5 ) | (d>>27);
		c += ((d & (a | b)) | (a & b)) + nt_buffer[9] +SQRT_2; c = (c<<9 ) | (c>>23);
		b += ((c & (d | a)) | (d & a)) + nt_buffer[13]+SQRT_2; b = (b<<13) | (b>>19);

		a += ((b & (c | d)) | (c & d)) + nt_buffer[2] +SQRT_2; a = (a<<3 ) | (a>>29);
		d += ((a & (b | c)) | (b & c)) + nt_buffer[6] +SQRT_2; d = (d<<5 ) | (d>>27);
		c += ((d & (a | b)) | (a & b)) + nt_buffer[10]+SQRT_2; c = (c<<9 ) | (c>>23);
		b += ((c & (d | a)) | (d & a)) + nt_buffer[14]+SQRT_2; b = (b<<13) | (b>>19);

		a += ((b & (c | d)) | (c & d)) + nt_buffer[3] +SQRT_2; a = (a<<3 ) | (a>>29);
		d += ((a & (b | c)) | (b & c)) + nt_buffer[7] +SQRT_2; d = (d<<5 ) | (d>>27);
		c += ((d & (a | b)) | (a & b)) + nt_buffer[11]+SQRT_2; c = (c<<9 ) | (c>>23);
		b += ((c & (d | a)) | (d & a)) +SQRT_2; b = (b<<13) | (b>>19);

		/* Round 3 */
		a += (d ^ c ^ b) + nt_buffer[0]  +  SQRT_3; a = (a << 3 ) | (a >> 29);
		d += (c ^ b ^ a) + nt_buffer[8]  +  SQRT_3; d = (d << 9 ) | (d >> 23);
		c += (b ^ a ^ d) + nt_buffer[4]  +  SQRT_3; c = (c << 11) | (c >> 21);
		b += (a ^ d ^ c) + nt_buffer[12] +  SQRT_3; b = (b << 15) | (b >> 17);

		a += (d ^ c ^ b) + nt_buffer[2]  +  SQRT_3; a = (a << 3 ) | (a >> 29);
		d += (c ^ b ^ a) + nt_buffer[10] +  SQRT_3; d = (d << 9 ) | (d >> 23);
		c += (b ^ a ^ d) + nt_buffer[6]  +  SQRT_3; c = (c << 11) | (c >> 21);
		b += (a ^ d ^ c) + nt_buffer[14] +  SQRT_3; b = (b << 15) | (b >> 17);

		a += (d ^ c ^ b) + nt_buffer[1]  +  SQRT_3; a = (a << 3 ) | (a >> 29);
		d += (c ^ b ^ a) + nt_buffer[9]  +  SQRT_3; d = (d << 9 ) | (d >> 23);
		c += (b ^ a ^ d) + nt_buffer[5]  +  SQRT_3; c = (c << 11) | (c >> 21);
		b += (a ^ d ^ c) + nt_buffer[13] +  SQRT_3; b = (b << 15) | (b >> 17);

		a += (d ^ c ^ b) + nt_buffer[3]  +  SQRT_3; a = (a << 3 ) | (a >> 29);
		d += (c ^ b ^ a) + nt_buffer[11] +  SQRT_3; d = (d << 9 ) | (d >> 23);
		c += (b ^ a ^ d) + nt_buffer[7]  +  SQRT_3; c = (c << 11) | (c >> 21);
		b += (a ^ d ^ c) +  SQRT_3; b = (b << 15) | (b >> 17);

		((unsigned int *)p16)[0] = a + INIT_A;
		((unsigned int *)p16)[1] = b + INIT_B;
		((unsigned int *)p16)[2] = c + INIT_C;
		((unsigned int *)p16)[3] = d + INIT_D;
	}
#endif
	return trunclen;
}

// Convert UTF-16LE to UTF-8. This is not optimised as it's
// only used in get_key() as of now.
// Non thread-safe version
UTF8 * utf16toutf8 (const UTF16* source) {
	static UTF8 ret_Key[PLAINTEXT_BUFFER_SIZE + 1];
	return utf16toutf8_r(ret_Key, PLAINTEXT_BUFFER_SIZE, source);
}

// Thread-safe version
UTF8 * utf16toutf8_r (UTF8 *dst, int dst_len, const UTF16* source) {
	UTF8 *tpt = dst;
	UTF8 *targetEnd = tpt + dst_len;
	while (*source) {
		UTF32 ch;
		unsigned short bytesToWrite = 0;
		const UTF32 byteMask = 0xBF;
		const UTF32 byteMark = 0x80;
		const UTF16* oldSource = source; /* In case we have to back out */
		ch = *source++;
		/* If we have a surrogate pair, convert to UTF32 first. */
		if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) {
			/* If the 16 bits following the high surrogate are in the source buffer... */
			if (source) {
				UTF32 ch2 = *source;
				/* If it's a low surrogate, convert to UTF32. */
				if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) {
					ch = ((ch - UNI_SUR_HIGH_START) << halfShift)
						+ (ch2 - UNI_SUR_LOW_START) + halfBase;
					++source;
				} else { /* it's an unpaired high surrogate */
					--source; /* return to the illegal value itself */
					fprintf(stderr, "warning, utf16toutf8 failed (illegal) - this is a bug in JtR\n");
					break;
				}
			} else { /* We don't have the 16 bits following the high surrogate. */
				--source; /* return to the high surrogate */
				fprintf(stderr, "warning, utf16toutf8 failed (no surrogate) - this is a bug in JtR\n");
				break;
			}
		}
		/* Figure out how many bytes the result will require */
		if (ch < (UTF32)0x80) {	     bytesToWrite = 1;
		} else if (ch < (UTF32)0x800) {     bytesToWrite = 2;
		} else if (ch < (UTF32)0x10000) {   bytesToWrite = 3;
		} else if (ch < (UTF32)0x110000) {  bytesToWrite = 4;
		} else {			    bytesToWrite = 3;
			ch = UNI_REPLACEMENT_CHAR;
		}

		tpt += bytesToWrite;
		if (tpt > targetEnd) {
			source = oldSource; /* Back up source pointer! */
			tpt -= bytesToWrite; break;
		}
		switch (bytesToWrite) { /* note: everything falls through. */
		case 4: *--tpt = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
		case 3: *--tpt = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
		case 2: *--tpt = (UTF8)((ch | byteMark) & byteMask); ch >>= 6;
		case 1: *--tpt =  (UTF8)(ch | firstByteMark[bytesToWrite]);
		}
		tpt += bytesToWrite;
	}
	*tpt = 0;
	return dst;
}

// Convert UTF-16LE to UTF-8 or ISO-8859-1 depending on --encoding=utf8 flag.
// This is not optimised as it's only used in get_key() as of now.
// Non thread-safe version
UTF8 * utf16toplain (const UTF16* source) {
	static UTF8 ret_Key[PLAINTEXT_BUFFER_SIZE + 1];
	return utf16toplain_r(ret_Key, PLAINTEXT_BUFFER_SIZE, source);
}

// Thread-safe version
UTF8 * utf16toplain_r (UTF8 *dst, int dst_len, const UTF16* source) {
	if (options.utf8)
		return utf16toutf8_r(dst, dst_len, source);
	else {
		UTF8 *tgt = dst;
		UTF8 *targetEnd = tgt + dst_len;

		while (*source && tgt < targetEnd)
			*tgt++ = CP_from_Unicode[*source++];
		*tgt = 0;
	}
	return dst;
}

/* Load the 'case-conversion' tables, and other translation table. */
int initUnicode() {
	unsigned i;
	unsigned char *cpU, *cpL;
	memset(utc2_upcase, 0, sizeof(utc2_upcase));
	memset(utc2_downcase, 0, sizeof(utc2_downcase));
	for (i = 0; arToUCDat[i]; i += 2)
		utc2_upcase[arToUCDat[i]] = arToUCDat[i+1];
	for (i = 0; arToLCDat[i]; i += 2)
		utc2_downcase[arToLCDat[i]] = arToLCDat[i+1];
	// set a 1 for any 'multi-char' converts.
	for (i = 0; uniMultiCase[i].Val; ++i)
		if (uniMultiCase[i].bIsUp2Low)
			utc2_downcase[uniMultiCase[i].Val] = 1;
		else
			utc2_upcase[uniMultiCase[i].Val] = 1;

	// Here we setup the 8-bit codepages we handle, and setup the mapping values into Unicode.
	for (i = 0; i < 128; ++i) {
		CP_to_Unicode[i] = i;
	}
	for (i = 128; i < 256; ++i) {
		if (options.cp1251)
			CP_to_Unicode[i] = CP1251_to_unicode_high128[i-128];
		else if (options.koi8_r)
			CP_to_Unicode[i] = KOI8_r_to_unicode_high128[i-128];
		else
			CP_to_Unicode[i] = i;
	}
	for (i = 0; i < 0x10000; ++i)
		CP_from_Unicode[i] = i;  // will truncate to lower 8 bits.
	if (options.cp1251 || options.koi8_r) {
		for (i = 0; i < 128; ++i) {
			if (options.cp1251)
				CP_from_Unicode[CP1251_to_unicode_high128[i]] = i+128;
			else
				CP_from_Unicode[KOI8_r_to_unicode_high128[i]] = i+128;
		}
	} else
		CP_from_Unicode[0x39c] = 0xB5;  // this is the only change to 'straight' 0 to 0xFF -> 0 to 0xFF.


	// first set ALL characters to have NO conversion.
	for (i = 0; i < 256; ++i)
		CP_up[i] = CP_down[i] = i;
	// standard case change for 7 bit characters (lower 128 bytes), for character sets.
	for (i = 'a'; i <= 'z'; ++i)
		CP_up[i] = (i^0x20);
	for (i = 'A'; i <= 'A'; ++i)
		CP_down[i] = (i^0x20);

	// now handle upper 128 byte values for casing.
	if (options.koi8_r) {
		cpU = (unsigned char*)CHARS_UPPER_KOI8_R; cpL = (unsigned char*)CHARS_LOWER_KOI8_R;
	} else if (options.cp1251) {
		cpU = (unsigned char*)CHARS_UPPER_CP1251; cpL = (unsigned char*)CHARS_LOWER_CP1251;
	} else {
		cpU = (unsigned char*)CHARS_UPPER_8859_1; cpL = (unsigned char*)CHARS_LOWER_8859_1;
	}
	for (i = 0; cpU[i]; ++i) {
		CP_down[(unsigned)cpU[i]] = cpL[i];
		CP_up[(unsigned)cpL[i]] = cpU[i];
	}
	return 0;
}

/* single char conversion inlines. Inlines vs macros, so that we get type 'safety'           */
/* NOTE these functions do NOT return multi UTF16 conversion characters, so they are         */
/* only 'partly' proper.  The plaintowcs_uc() and  plaintowcs_lc() do full conversions as    */
/* does the utf16_lc() and utf16_uc().  Full conversion uses the utc_*case[] arrays, but  it */
/* also uses the 1 UTC2 to multi UTC2 lookup table to do things 'properly'.                  */
//_inline unsigned char low2up_ascii(unsigned char c) {if (utc2_upcase[c]) {return utc2_upcase[c]; return c; }
//_inline unsigned char up2low_ascii(unsigned char c) {if (utc2_downcase[c]) return utc2_downcase[c]; return c; }
//_inline UTF16 low2up_u16(UTF16 w) {if (utc2_upcase[w]) return utc2_upcase[w]; return w; }
//_inline UTF16 up2low_u16(UTF16 w) {if (utc2_downcase[w]) return utc2_downcase[w]; return w; }

/* Convert to UTF-16LE from UTF-8 or ISO-8859-1 depending on --encoding=utf8 flag, and upcase/lowcase at same time */
//int plaintowcs_uc(UTF16 * dst, unsigned int maxdstlen, const UTF8 * src, unsigned int srclen) {
//	return 0;
//}
//int plaintowcs_lc(UTF16 * dst, unsigned int maxdstlen, const UTF8 * src, unsigned int srclen) {
//	return 0;
//}
int utf16_lc(UTF16 *dst, unsigned dst_len, const UTF16 *src, unsigned src_len) {
	int i, j = 0;
	UTF16 cur_src;
	for (i = 0; i < src_len; ++i, ++j, ++src, ++dst) {
		if (j == dst_len) {
			*dst = 0;
			return -j;
		}
		cur_src = BE_FIX(*src);
		if (utc2_downcase[ cur_src ] == 0)
			*dst = cur_src;
		else {
			if (utc2_downcase[ cur_src ] & 0xFFFE) {
				*dst = utc2_downcase[ cur_src ];
			}
			else {
				// multi-byte.
				int k, l;
				for (k = 0; uniMultiCase[k].Val; ++k) {
					if (uniMultiCase[k].Val == cur_src) {
						if (j + uniMultiCase[k].Cnt >= dst_len) {
							*dst = 0;
							return -j;
						}
						for (l = 0; l < uniMultiCase[k].Cnt; ++l)
							dst[l] = uniMultiCase[k].ToVals[l];
						dst += uniMultiCase[k].Cnt-1;
						j += uniMultiCase[k].Cnt-1;
						break;
					}
				}
			}
		}
	}
	*dst = 0;
	return j;
}

int utf16_uc(UTF16 *dst, unsigned dst_len, const UTF16 *src, unsigned src_len) {
	int i, j = 0;
	UTF16 cur_src;
	for (i = 0; i < src_len; ++i, ++j, ++src, ++dst) {
		if (j == dst_len) {
			*dst = 0;
			return -j;
		}
		cur_src = BE_FIX(*src);
		if (utc2_upcase[ cur_src ] == 0)
			*dst = cur_src;
		else {
			if (utc2_upcase[ cur_src ] & 0xFFFE) {
				*dst = utc2_upcase[ cur_src ];
			}
			else {
				// multi-byte.
				int k, l;
				for (k = 0; uniMultiCase[k].Val; ++k) {
					if (uniMultiCase[k].Val == cur_src) {
						if (j + uniMultiCase[k].Cnt >= dst_len) {
							*dst = 0;
							return -j;
						}
						for (l = 0; l < uniMultiCase[k].Cnt; ++l)
							dst[l] = uniMultiCase[k].ToVals[l];
						dst += uniMultiCase[k].Cnt-1;
						j += uniMultiCase[k].Cnt-1;
						break;
					}
				}
			}
		}
	}
	*dst = 0;
	return j;
}

int utf8_lc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len) {
	UTF16 tmp16[512+1], tmp16l[512+1]; // yes, short, but this is 'long enough' for john.
	int utf16len, i;

	if (!options.utf8) {
		if (dst_len <= src_len)
			src_len = dst_len - 1;
		for (i = 0; i < src_len; ++i) {
			*dst++ = CP_down[*src++];
		}
		*dst = 0;
		return src_len;
	}
	// too big for our function.  Someone needs to write one for longer strings.
	if (src_len > 512*3)
		return 0;
	utf16len = plaintowcs(tmp16, 512, src, src_len);
	if (utf16len <= 0)
		return 0;
	utf16len = utf16_lc(tmp16l, 512, tmp16, utf16len);
	if (utf16len <= 0)
		return 0;
	utf16toplain_r(dst, dst_len, tmp16l);
	return strlen((char*)dst);
}

int utf8_uc(UTF8 *dst, unsigned dst_len, const UTF8 *src, unsigned src_len) {
	UTF16 tmp16[512+1], tmp16u[512+1]; // yes, short, but this is 'long enough' for john.
	int utf16len, i;
	int OxDF_double_S = 0;

	if (!options.utf8) {
		int len;
		if (dst_len < src_len)
			src_len = dst_len;
		len=src_len;
		if (options.iso8859_1 || options.encoding_7_bit)
			OxDF_double_S = 1;

		for (i = 0; i < src_len; ++i) {
			if (OxDF_double_S && *src == 0xDF) { // this goes out as 2 characters.
				++len;
				if (len > dst_len) {
					return 0;
				}
				*dst++ = 'S';
				*dst++ = 'S';
				++src;
			}
			else
				*dst++ = CP_up[*src++];
		}
		*dst = 0;
		return len;
	}
	// too big for our function.  Someone needs to write one for longer strings.
	if (src_len > 512*3)
		return 0;
	utf16len = plaintowcs(tmp16, 512, src, src_len);
	if (utf16len <= 0)
		return 0;
	utf16len = utf16_uc(tmp16u, 512, tmp16, utf16len);
	if (utf16len <= 0)
		return 0;
	utf16toplain_r(dst, dst_len, tmp16u);
	return strlen((char*)dst);
}
