Rebase

CMakeLists.txt

@@ -45,6 +45,40 @@ function (die msg)
   message(FATAL_ERROR "${BoldRed}${msg}${ColourReset}")
 endfunction ()
 
+if (NOT ${ARCH} STREQUAL "")
+	string(SUBSTRING ${ARCH} 0 3 IS_ARM)
+	string(TOLOWER ${IS_ARM} IS_ARM)
+
+	if (${IS_ARM} STREQUAL "arm")
+		string(SUBSTRING ${ARCH} 0 5 ARM_TEST)
+		string(TOLOWER ${ARM_TEST} ARM_TEST)
+	
+		if (${ARM_TEST} STREQUAL "armv6")
+			set(ARM6 1)
+		else()
+			set(ARM6 0)
+		endif()
+	   
+		if (${ARM_TEST} STREQUAL "armv7")
+			set(ARM7 1)
+		else()
+			set(ARM7 0)
+		endif()	
+	endif()
+endif()
+
+if(WIN32 OR ARM7 OR ARM6)
+	set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
+	set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
+endif()
+
+# set this to 0 if per-block checkpoint needs to be disabled
+set(PER_BLOCK_CHECKPOINT 1)
+
+if(PER_BLOCK_CHECKPOINT)
+  add_definitions("-DPER_BLOCK_CHECKPOINT")
+endif()
+
 list(INSERT CMAKE_MODULE_PATH 0
   "${CMAKE_SOURCE_DIR}/cmake")
 
@@ -127,16 +161,16 @@ endif()
 option(STATIC "Link libraries statically" ${DEFAULT_STATIC})
 
 if(MINGW)
-  get_filename_component(msys2_install_path "[HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\MSYS2 ${ARCH_WIDTH}bit;InstallLocation]" ABSOLUTE)
+  string(REGEX MATCH "^[^/]:/[^/]*" msys2_install_path "${CMAKE_C_COMPILER}")
+  message(STATUS "MSYS location: ${msys2_install_path}")
   set(CMAKE_INCLUDE_PATH "${msys2_install_path}/mingw${ARCH_WIDTH}/include")
   # This is necessary because otherwise CMake will make Boost libraries -lfoo
   # rather than a full path. Unfortunately, this makes the shared libraries get
   # linked due to a bug in CMake which misses putting -static flags around the
   # -lfoo arguments.
-  list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES
+  set(DEFLIB ${msys2_install_path}/mingw${ARCH_WIDTH}/lib)
-    "${msys2_install_path}/mingw${ARCH_WIDTH}/lib")
+  list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES ${DEFLIB})
-  list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES
+  list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES ${DEFLIB})
-    "${msys2_install_path}/mingw${ARCH_WIDTH}/lib")
 endif()
 
 if(STATIC)
@@ -158,18 +192,47 @@ if (DEFINED ENV{DATABASE})
 else()
   message(STATUS "Could not find DATABASE in env (not required unless you want to change database type from default: ${DATABASE})")
 endif()
+
+set(BERKELEY_DB 0)
 if (DATABASE STREQUAL "lmdb")
   set(BLOCKCHAIN_DB DB_LMDB)
+  
+  # temporarily allow mingw to compile with berkeley_db,
+  # regardless if building static or not
+  if(NOT STATIC OR MINGW)
+	  find_package(BerkeleyDB)
+		
+	  if(NOT BERKELEY_DB_LIBRARIES)
+	    message(STATUS "BerkeleyDB not found and has been disabled.")
+	  else()
+	    message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
+		if(BERKELEY_DB_LIBRARIES)
+		  message(STATUS "Found BerkeleyDB shared library")
+		  set(BDB_STATIC false CACHE BOOL "BDB Static flag")
+		  set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
+		  set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
+		  set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
+		  set(BERKELEY_DB 1)
+		else()
+		  message(STATUS "Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
+		endif()
+	  endif()
+  endif()
 elseif (DATABASE STREQUAL "memory")
   set(BLOCKCHAIN_DB DB_MEMORY)
 else()
   die("Invalid database type: ${DATABASE}")
 endif()
 
+if(BERKELEY_DB)
+  add_definitions("-DBERKELEY_DB")
+endif()
+	
 add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}")
 
 if (UNIX AND NOT APPLE)
   # Note that at the time of this writing the -Wstrict-prototypes flag added below will make this fail
+  set(THREADS_PREFER_PTHREAD_FLAG ON)
   find_package(Threads)
 endif()
 
@@ -194,7 +257,7 @@ include_directories(external/rapidjson)
 include_directories(${LMDB_INCLUDE})
 
 # Final setup for Berkeley DB
-if (NOT STATIC)
+if (BERKELEY_DB)
   include_directories(${BDB_INCLUDE})
 endif()
 
@@ -210,10 +273,15 @@ if(MSVC)
   include_directories(SYSTEM src/platform/msc)
 else()
   set(ARCH native CACHE STRING "CPU to build for: -march value or default")
-  if(ARCH STREQUAL "default")
+  # -march=armv7-a conflicts with -mcpu=cortex-a7
+  if(ARCH STREQUAL "default" OR ARM7)
     set(ARCH_FLAG "")
   else()
-    set(ARCH_FLAG "-march=${ARCH}")
+    if(ARCH STREQUAL "x86_64")
+      set(ARCH_FLAG "-march=x86-64")
+    else()
+       set(ARCH_FLAG "-march=${ARCH}")
+    endif()
   endif()
   set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
   if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
@@ -256,14 +324,18 @@ else()
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} -maes")
   endif()
 
-  string(SUBSTRING ${ARCH} 0 3 ARM_TEST)
+  if(ARM6)
-  string(TOLOWER ${ARM_TEST} ARM_TEST)
+    message(STATUS "Setting ARM6 C and C++ flags")
-  if(${ARM_TEST} STREQUAL "arm")
-    message(STATUS "Setting ARM C and C++ flags")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfpu=vfp -mfloat-abi=hard")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=vfp -mfloat-abi=hard")
   endif()
 
+  if(ARM7)
+    message(STATUS "Setting ARM7 C and C++ flags")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -mcpu=cortex-a7 -mfloat-abi=hard -mfpu=vfpv4 -funsafe-math-optimizations -mtune=cortex-a7")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -mcpu=cortex-a7 -mfloat-abi=hard -mfpu=vfpv4 -funsafe-math-optimizations -mtune=cortex-a7")
+  endif()
+
   if(APPLE)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_TR1_TUPLE=0")
   endif()
@@ -341,7 +413,7 @@ elseif(APPLE OR FREEBSD)
   set(EXTRA_LIBRARIES "")
 elseif(NOT MSVC)
   find_library(RT rt)
-  set(EXTRA_LIBRARIES ${RT} ${PTHREAD} ${DL})
+  set(EXTRA_LIBRARIES ${RT} ${DL})
 endif()
 
 include(version.cmake)

Makefile

@@ -58,6 +58,14 @@ release-all:
 	mkdir -p build/release
 	cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
 
+release-arm6:
+	mkdir -p build/release
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv6zk" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+
+release-arm7:
+	mkdir -p build/release
+	cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
+	
 release-static: release-static-64
 
 release-static-64:
@@ -68,10 +76,6 @@ release-static-32:
 	mkdir -p build/release
 	cd build/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
 
-release-static-arm6:
-	mkdir -p build/release
-	cd build/release && cmake -D STATIC=ON -D ARCH="armv6zk" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
-
 release-static-win64:
 	mkdir -p build/release
 	cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ../.. && $(MAKE)

README.i18n

@@ -0,0 +1,45 @@
+The Monero command line tools can be translated in various languages.
+In order to use the same translation workflow as the future GUI, they
+use Qt Linguist translation files. However, to avoid the dependencies
+on Qt this normally implies, they use a custom loader to read those
+files at runtime. In order to update, or build translations files, you
+do need to have Qt tools installed, however. For translating, you need
+either the Qt Linguist GUI, or another tool that supports Qt ts files,
+such as Transifex. To run, you do not need anything Qt.
+
+To update ts files after changing source code:
+
+  ./utils/translations/update-translations.sh
+
+To add a new language, eg Spanish (ISO code es):
+
+  cp transations/monero.ts transations/monero_es.ts
+
+To edit translations for Spanish:
+
+  linguist translations/monero_es.ts
+
+To build translations after modiying them:
+
+  ./utils/translations/build-translations.sh
+
+To test a translation:
+
+  LANG=es ./build/release/bin/simplewallet
+
+To add new translatable sources in the source:
+
+  Use the tr(string) function if possible. If the code is in a class,
+  and this class doesn't already have a tr() static function, add one,
+  which uses a context named after what lupdate uses for the context,
+  usually the fully qualified class name (eg, cryptonote::simple_wallet).
+  If you need to use tr in code that's not in a class, you can use the
+  fully qualified version (eg, simple_wallet::tr) of the one matching
+  the context you want.
+  Use QT_TRANSLATE_NOOP(string) if you want to specify a context manually.
+
+If you're getting messages of the form:
+
+  Class 'cryptonote::simple_wallet' lacks Q_OBJECT macro
+
+all is fine, we don't actually need that here.

README.md

@@ -134,7 +134,7 @@ cd build
 ```
 * If you are on a 64-bit system, run:
 ```
-cmake -G "MSYS Makefiles" -D CMAKE_BUILD_TYPE=Release -D ARCH="x86_64" -D BUILD_64=ON -D CMAKE_TOOLCHAIN_FILE=../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ..
+cmake -G "MSYS Makefiles" -D CMAKE_BUILD_TYPE=Release -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_TOOLCHAIN_FILE=../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ..
 ```
 * If you are on a 32-bit system, run:
 ```
@@ -167,3 +167,8 @@ Dependencies: Doxygen 1.8.0 or later, Graphviz 2.28 or later (optional).
 * To build, change to the root of the source code directory, and run `doxygen Doxyfile`
 * If you have installed Graphviz, you can also generate in-doc diagrams by instead running `HAVE_DOT=YES doxygen Doxyfile`
 * The output will be built in doc/html/
+
+## Internationalization
+
+See README.i18n
+

contrib/epee/include/console_handler.h

@@ -274,14 +274,15 @@ namespace epee
         }
 
         std::string command;
-        if(!m_stdin_reader.get_line(command))
+        bool get_line_ret = m_stdin_reader.get_line(command);
-        {
-          LOG_PRINT("Failed to read line.", LOG_LEVEL_0);
-        }
         if (m_stdin_reader.eos())
         {
           break;
         }
+        if (!get_line_ret)
+        {
+          LOG_PRINT("Failed to read line.", LOG_LEVEL_0);
+        }
         string_tools::trim(command);
 
         LOG_PRINT_L2("Read command: " << command);
@@ -303,7 +304,8 @@ namespace epee
           std::cout << usage;
         }
       }
-      exit_handler();
+      if (exit_handler)
+        exit_handler();
       return true;
       CATCH_ENTRY_L0("console_handler", false);
     }

contrib/epee/include/misc_log_ex.h

@@ -861,7 +861,8 @@ namespace log_space
       std::string::size_type a = m_default_log_file.rfind('.');
       if ( a != std::string::npos )
         m_default_log_file.erase( a, m_default_log_file.size());
-      m_default_log_file += ".log";
+      if ( ! m_default_log_file.empty() )
+        m_default_log_file += ".log";
 
       return true;
     }

external/db_drivers/CMakeLists.txt

@@ -31,24 +31,4 @@
 message(STATUS "Using ${ARCH_WIDTH}-bit LMDB from source tree")
 add_subdirectory(liblmdb${ARCH_WIDTH})
 set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb${ARCH_WIDTH}" CACHE STRING "LMDB Include path")
-
 set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name")
-
-if (NOT STATIC)
-  find_package(BerkeleyDB)
-
-  if(NOT BERKELEY_DB_LIBRARIES)
-    die("BerkeleyDB not found.  At this time it should be installed in your system for a non-static build.")
-  else()
-    message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
-    if(BERKELEY_DB_LIBRARIES)
-      message(STATUS "Found BerkeleyDB shared library")
-      set(BDB_STATIC false CACHE BOOL "BDB Static flag")
-      set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
-      set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
-      set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
-    else()
-      die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
-    endif()
-  endif()
-endif()

external/db_drivers/liblmdb32/CHANGES

@@ -1,5 +1,33 @@
 LMDB 0.9 Change Log
 
+LMDB 0.9.15 Release (2015/06/19)
+	Fix txn init (ITS#7961,#7987)
+	Fix MDB_PREV_DUP (ITS#7955,#7671)
+	Fix compact of empty env (ITS#7956)
+	Fix mdb_copy file mode
+	Fix mdb_env_close() after failed mdb_env_open()
+	Fix mdb_rebalance collapsing root (ITS#8062)
+	Fix mdb_load with large values (ITS#8066)
+	Fix to retry writes on EINTR (ITS#8106)
+	Fix mdb_cursor_del on empty DB (ITS#8109)
+	Fix MDB_INTEGERDUP key compare (ITS#8117)
+	Fix error handling (ITS#7959,#8157,etc.)
+	Fix race conditions (ITS#7969,7970)
+	Added workaround for fdatasync bug in ext3fs
+	Build
+		Don't use -fPIC for static lib
+		Update .gitignore (ITS#7952,#7953)
+		Cleanup for "make test" (ITS#7841), "make clean", mtest*.c
+		Misc. Android/Windows cleanup
+	Documentation
+		Fix MDB_APPEND doc
+		Fix MDB_MAXKEYSIZE doc (ITS#8156)
+		Fix mdb_cursor_put,mdb_cursor_del EACCES description
+		Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021)
+		Clarify MDB_WRITEMAP doc (ITS#8021)
+		Clarify mdb_env_open doc
+		Clarify mdb_dbi_open doc
+
 LMDB 0.9.14 Release (2014/09/20)
 	Fix to support 64K page size (ITS#7713)
 	Fix to persist decreased as well as increased mapsizes (ITS#7789)

external/db_drivers/liblmdb32/CMakeLists.txt

@@ -26,6 +26,13 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+if(FREEBSD)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMDB_DSYNC=O_SYNC")
+endif()
+
+# pass the VL32 flag so that we actually invoke the VL32 extensions
+set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVL32")
+
 set (lmdb_sources
 mdb.c
 midl.c)

external/db_drivers/liblmdb32/COPYRIGHT

@@ -1,4 +1,4 @@
-Copyright 2011-2014 Howard Chu, Symas Corp.
+Copyright 2011-2015 Howard Chu, Symas Corp.
 All rights reserved.
 
 Redistribution and use in source and binary forms, with or without

external/db_drivers/liblmdb32/Makefile

@@ -11,6 +11,7 @@
 # - MDB_USE_POSIX_SEM
 # - MDB_DSYNC
 # - MDB_FDATASYNC
+# - MDB_FDATASYNC_WORKS
 # - MDB_USE_PWRITEV
 #
 # There may be other macros in mdb.c of interest. You should
@@ -42,18 +43,18 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
 	for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done
 
 clean:
-	rm -rf $(PROGS) *.[ao] *.so *~ testdb
+	rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb
 
 test:	all
-	mkdir testdb
+	rm -rf testdb && mkdir testdb
 	./mtest && ./mdb_stat testdb
 
 liblmdb.a:	mdb.o midl.o
 	ar rs $@ mdb.o midl.o
 
-liblmdb.so:	mdb.o midl.o
+liblmdb.so:	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
-	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS)
+	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
 
 mdb_stat: mdb_stat.o liblmdb.a
 mdb_copy: mdb_copy.o liblmdb.a
@@ -67,10 +68,16 @@ mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
 
 mdb.o: mdb.c lmdb.h midl.h
-	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
 
 midl.o: midl.c midl.h
-	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c midl.c
+
+mdb.lo: mdb.c lmdb.h midl.h
+	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c -o $@
+
+midl.lo: midl.c midl.h
+	$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c -o $@
 
 %:	%.o
 	$(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@

external/db_drivers/liblmdb32/lmdb.h

@@ -119,7 +119,7 @@
  *
  *	@author	Howard Chu, Symas Corporation.
  *
- *	@copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved.
+ *	@copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted only as authorized by the OpenLDAP
@@ -184,7 +184,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	14
+#define MDB_VERSION_PATCH	15
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -194,7 +194,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"September 20, 2014"
+#define MDB_VERSION_DATE	"June 19, 2015"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"
@@ -296,12 +296,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_REVERSEKEY	0x02
 	/** use sorted duplicates */
 #define MDB_DUPSORT		0x04
-	/** numeric keys in native byte order.
+	/** numeric keys in native byte order: either unsigned int or size_t.
 	 *  The keys must all be of the same size. */
 #define MDB_INTEGERKEY	0x08
 	/** with #MDB_DUPSORT, sorted dup items have fixed size */
 #define MDB_DUPFIXED	0x10
-	/** with #MDB_DUPSORT, dups are numeric in native byte order */
+	/** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
 #define MDB_INTEGERDUP	0x20
 	/** with #MDB_DUPSORT, use reverse string dups */
 #define MDB_REVERSEDUP	0x40
@@ -516,8 +516,8 @@ int  mdb_env_create(MDB_env **env);
 	 *		and uses fewer mallocs, but loses protection from application bugs
 	 *		like wild pointer writes and other bad updates into the database.
 	 *		Incompatible with nested transactions.
-	 *		Processes with and without MDB_WRITEMAP on the same environment do
+	 *		Do not mix processes with and without MDB_WRITEMAP on the same
-	 *		not cooperate well.
+	 *		environment.  This can defeat durability (#mdb_env_sync etc).
 	 *	<li>#MDB_NOMETASYNC
 	 *		Flush system buffers to disk only once per transaction, omit the
 	 *		metadata flush. Defer that until the system flushes files to disk,
@@ -588,8 +588,8 @@ int  mdb_env_create(MDB_env **env);
 	 *		reserved in that case.
 	 *		This flag may be changed at any time using #mdb_env_set_flags().
 	 * </ul>
-	 * @param[in] mode The UNIX permissions to set on created files. This parameter
+	 * @param[in] mode The UNIX permissions to set on created files and semaphores.
-	 * is ignored on Windows.
+	 * This parameter is ignored on Windows.
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
@@ -698,7 +698,8 @@ int  mdb_env_info(MDB_env *env, MDB_envinfo *stat);
 	 * Data is always written to disk when #mdb_txn_commit() is called,
 	 * but the operating system may keep it buffered. LMDB always flushes
 	 * the OS buffers upon commit as well, unless the environment was
-	 * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC.
+	 * opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is
+	 * not valid if the environment was opened with #MDB_RDONLY.
 	 * @param[in] env An environment handle returned by #mdb_env_create()
 	 * @param[in] force If non-zero, force a synchronous flush.  Otherwise
 	 *  if the environment has the #MDB_NOSYNC flag set the flushes
@@ -706,6 +707,7 @@ int  mdb_env_info(MDB_env *env, MDB_envinfo *stat);
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
+	 *	<li>EACCES - the environment is read-only.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 *	<li>EIO - an error occurred during synchronization.
 	 * </ul>
@@ -1019,15 +1021,17 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 * The database handle may be discarded by calling #mdb_dbi_close().
 	 * The old database handle is returned if the database was already open.
 	 * The handle may only be closed once.
+	 *
 	 * The database handle will be private to the current transaction until
 	 * the transaction is successfully committed. If the transaction is
 	 * aborted the handle will be closed automatically.
-	 * After a successful commit the
+	 * After a successful commit the handle will reside in the shared
-	 * handle will reside in the shared environment, and may be used
+	 * environment, and may be used by other transactions.
-	 * by other transactions. This function must not be called from
+	 *
-	 * multiple concurrent transactions. A transaction that uses this function
+	 * This function must not be called from multiple concurrent
-	 * must finish (either commit or abort) before any other transaction may
+	 * transactions in the same process. A transaction that uses
-	 * use this function.
+	 * this function must finish (either commit or abort) before
+	 * any other transaction in the process may use this function.
 	 *
 	 * To use named databases (with name != NULL), #mdb_env_set_maxdbs()
 	 * must be called before opening the environment.  Database names
@@ -1048,9 +1052,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		keys may have multiple data items, stored in sorted order.) By default
 	 *		keys must be unique and may have only a single data item.
 	 *	<li>#MDB_INTEGERKEY
-	 *		Keys are binary integers in native byte order. Setting this option
+	 *		Keys are binary integers in native byte order, either unsigned int
-	 *		requires all keys to be the same size, typically sizeof(int)
+	 *		or size_t, and will be sorted as such.
-	 *		or sizeof(size_t).
+	 *		The keys must all be of the same size.
 	 *	<li>#MDB_DUPFIXED
 	 *		This flag may only be used in combination with #MDB_DUPSORT. This option
 	 *		tells the library that the data items for this database are all the same
@@ -1058,8 +1062,8 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
 	 *		cursor operations may be used to retrieve multiple items at once.
 	 *	<li>#MDB_INTEGERDUP
-	 *		This option specifies that duplicate data items are also integers, and
+	 *		This option specifies that duplicate data items are binary integers,
-	 *		should be sorted as such.
+	 *		similar to #MDB_INTEGERKEY keys.
 	 *	<li>#MDB_REVERSEDUP
 	 *		This option specifies that duplicate data items should be compared as
 	 *		strings in reverse order.
@@ -1270,10 +1274,9 @@ int  mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
 	 *		LMDB does nothing else with this memory, the caller is expected
 	 *		to modify all of the space requested.
 	 *	<li>#MDB_APPEND - append the given key/data pair to the end of the
-	 *		database. No key comparisons are performed. This option allows
+	 *		database. This option allows fast bulk loading when keys are
-	 *		fast bulk loading when keys are already known to be in the
+	 *		already known to be in the correct order. Loading unsorted keys
-	 *		correct order. Loading unsorted keys with this flag will cause
+	 *		with this flag will cause a #MDB_KEYEXIST error.
-	 *		data corruption.
 	 *	<li>#MDB_APPENDDUP - as above, but for sorted dup data.
 	 * </ul>
 	 * @return A non-zero error value on failure and 0 on success. Some possible
@@ -1449,7 +1452,7 @@ int  mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 * <ul>
 	 *	<li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
 	 *	<li>#MDB_TXN_FULL - the transaction has too many dirty pages.
-	 *	<li>EACCES - an attempt was made to modify a read-only database.
+	 *	<li>EACCES - an attempt was made to write in a read-only transaction.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 * </ul>
 	 */
@@ -1469,7 +1472,7 @@ int  mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
-	 *	<li>EACCES - an attempt was made to modify a read-only database.
+	 *	<li>EACCES - an attempt was made to write in a read-only transaction.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 * </ul>
 	 */

external/db_drivers/liblmdb32/mdb.c

@@ -5,7 +5,7 @@
  *	BerkeleyDB API, but much simplified.
  */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -79,6 +79,14 @@ extern int cacheflush(char *addr, int nbytes, int cache);
 #define CACHEFLUSH(addr, bytes, cache)
 #endif
 
+#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS)
+/** fdatasync is broken on ext3/ext4fs on older kernels, see
+ *	description in #mdb_env_open2 comments. You can safely
+ *	define MDB_FDATASYNC_WORKS if this code will only be run
+ *	on kernels 3.6 and newer.
+ */
+#define	BROKEN_FDATASYNC
+#endif
 
 #include <errno.h>
 #include <limits.h>
@@ -438,12 +446,17 @@ static txnid_t mdb_debug_start;
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 1
 
-	/**	@brief The max size of a key we can write, or 0 for dynamic max.
+	/**	@brief The max size of a key we can write, or 0 for computed max.
 	 *
-	 *	Define this as 0 to compute the max from the page size.  511
+	 *	This macro should normally be left alone or set to 0.
-	 *	is default for backwards compat: liblmdb <= 0.9.10 can break
+	 *	Note that a database with big keys or dupsort data cannot be
-	 *	when modifying a DB with keys/dupsort data bigger than its max.
+	 *	reliably modified by a liblmdb which uses a smaller max.
-	 *	#MDB_DEVEL sets the default to 0.
+	 *	The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
+	 *
+	 *	Other values are allowed, for backwards compat.  However:
+	 *	A value bigger than the computed max can break if you do not
+	 *	know what you are doing, and liblmdb <= 0.9.10 can break when
+	 *	modifying a DB with keys/dupsort data bigger than its max.
 	 *
 	 *	Data items in an #MDB_DUPSORT database are also limited to
 	 *	this size, since they're actually keys of a sub-DB.  Keys and
@@ -580,11 +593,11 @@ typedef struct MDB_rxbody {
 	 *	started from so we can avoid overwriting any data used in that
 	 *	particular version.
 	 */
-	txnid_t		mrb_txnid;
+	volatile txnid_t		mrb_txnid;
 	/** The process ID of the process owning this reader txn. */
-	MDB_PID_T	mrb_pid;
+	volatile MDB_PID_T	mrb_pid;
 	/** The thread ID of the thread owning this txn. */
-	MDB_THR_T	mrb_tid;
+	volatile MDB_THR_T	mrb_tid;
 } MDB_rxbody;
 
 	/** The actual reader record, with cacheline padding. */
@@ -632,12 +645,12 @@ typedef struct MDB_txbody {
 		 *	This is recorded here only for convenience; the value can always
 		 *	be determined by reading the main database meta pages.
 		 */
-	txnid_t		mtb_txnid;
+	volatile txnid_t		mtb_txnid;
 		/** The number of slots that have been used in the reader table.
 		 *	This always records the maximum count, it is not decremented
 		 *	when readers release their slots.
 		 */
-	unsigned	mtb_numreaders;
+	volatile unsigned	mtb_numreaders;
 } MDB_txbody;
 
 	/** The actual reader table definition. */
@@ -898,7 +911,7 @@ typedef struct MDB_meta {
 		/** Stamp identifying this as an LMDB file. It must be set
 		 *	to #MDB_MAGIC. */
 	uint32_t	mm_magic;
-		/** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */
+		/** Version number of this file. Must be set to #MDB_DATA_VERSION. */
 	uint32_t	mm_version;
 	void		*mm_address;		/**< address for fixed mapping */
 	size_t		mm_mapsize;			/**< size of mmap region */
@@ -908,7 +921,7 @@ typedef struct MDB_meta {
 	/** Any persistent environment flags. @ref mdb_env */
 #define	mm_flags	mm_dbs[0].md_flags
 	pgno_t		mm_last_pg;			/**< last used page in file */
-	txnid_t		mm_txnid;			/**< txnid that committed this page */
+	volatile txnid_t	mm_txnid;	/**< txnid that committed this page */
 } MDB_meta;
 
 	/** Buffer for a stack-allocated meta page.
@@ -1103,6 +1116,8 @@ struct MDB_env {
 #define	MDB_ENV_ACTIVE	0x20000000U
 	/** me_txkey is set */
 #define	MDB_ENV_TXKEY	0x10000000U
+	/** fdatasync is unreliable */
+#define	MDB_FSYNCONLY	0x08000000U
 	uint32_t 	me_flags;		/**< @ref mdb_env */
 	unsigned int	me_psize;	/**< DB page size, inited from me_os_psize */
 	unsigned int	me_os_psize;	/**< OS page size, from #GET_PAGESIZE */
@@ -1242,6 +1257,13 @@ static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
 /** @endcond */
 
+/** Compare two items pointing at size_t's of unknown alignment. */
+#ifdef MISALIGNED_OK
+# define mdb_cmp_clong mdb_cmp_long
+#else
+# define mdb_cmp_clong mdb_cmp_cint
+#endif
+
 #ifdef _WIN32
 static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
@@ -1323,7 +1345,7 @@ mdb_strerror(int err)
 	buf[0] = 0;
 	FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
 		FORMAT_MESSAGE_IGNORE_INSERTS,
-		NULL, err, 0, ptr, sizeof(buf), pad);
+		NULL, err, 0, ptr, sizeof(buf), (va_list *)pad);
 	return ptr;
 #else
 	return strerror(err);
@@ -1555,7 +1577,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 int
 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 {
-	return txn->mt_dbxs[dbi].md_dcmp(a, b);
+	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
+#if UINT_MAX < SIZE_MAX
+	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
+		dcmp = mdb_cmp_clong;
+#endif
+	return dcmp(a, b);
 }
 
 /** Allocate memory for a page.
@@ -2323,6 +2350,8 @@ int
 mdb_env_sync(MDB_env *env, int force)
 {
 	int rc = 0;
+	if (env->me_flags & MDB_RDONLY)
+		return EACCES;
 	if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
 		if (env->me_flags & MDB_WRITEMAP) {
 			int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
@@ -2334,6 +2363,12 @@ mdb_env_sync(MDB_env *env, int force)
 				rc = ErrCode();
 #endif
 		} else {
+#ifdef BROKEN_FDATASYNC
+			if (env->me_flags & MDB_FSYNCONLY) {
+				if (fsync(env->me_fd))
+					rc = ErrCode();
+			} else
+#endif
 			if (MDB_FDATASYNC(env->me_fd))
 				rc = ErrCode();
 		}
@@ -2489,15 +2524,11 @@ mdb_txn_renew0(MDB_txn *txn)
 	MDB_env *env = txn->mt_env;
 	MDB_txninfo *ti = env->me_txns;
 	MDB_meta *meta;
-	unsigned int i, nr;
+	unsigned int i, nr, flags = txn->mt_flags;
 	uint16_t x;
 	int rc, new_notls = 0;
 
-	/* Setup db info */
+	if ((flags &= MDB_TXN_RDONLY) != 0) {
-	txn->mt_numdbs = env->me_numdbs;
-	txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
-
-	if (txn->mt_flags & MDB_TXN_RDONLY) {
 		if (!ti) {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
@@ -2543,10 +2574,14 @@ mdb_txn_renew0(MDB_txn *txn)
 					return rc;
 				}
 			}
-			txn->mt_txnid = r->mr_txnid = ti->mti_txnid;
+			do /* LY: Retry on a race, ITS#7970. */
+				r->mr_txnid = ti->mti_txnid;
+			while(r->mr_txnid != ti->mti_txnid);
+			txn->mt_txnid = r->mr_txnid;
 			txn->mt_u.reader = r;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}
+		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
 	} else {
 		if (ti) {
 			LOCK_MUTEX_W(env);
@@ -2562,6 +2597,9 @@ mdb_txn_renew0(MDB_txn *txn)
 		if (txn->mt_txnid == mdb_debug_start)
 			mdb_debug = 1;
 #endif
+		txn->mt_child = NULL;
+		txn->mt_loose_pgs = NULL;
+		txn->mt_loose_count = 0;
 		txn->mt_dirty_room = MDB_IDL_UM_MAX;
 		txn->mt_u.dirty_list = env->me_dirty_list;
 		txn->mt_u.dirty_list[0].mid = 0;
@@ -2578,6 +2616,10 @@ mdb_txn_renew0(MDB_txn *txn)
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
 
+	txn->mt_flags = flags;
+
+	/* Setup db info */
+	txn->mt_numdbs = env->me_numdbs;
 	for (i=2; i<txn->mt_numdbs; i++) {
 		x = env->me_dbflags[i];
 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
@@ -2643,18 +2685,16 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
 		}
 		tsize = sizeof(MDB_ntxn);
 	}
-	size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1);
+	size = tsize;
 	if (!(flags & MDB_RDONLY)) {
 		if (!parent) {
-			txn = env->me_txn0;
+			txn = env->me_txn0;	/* just reuse preallocated write txn */
-			txn->mt_flags = 0;
 			goto ok;
 		}
+		/* child txns use own copy of cursors */
 		size += env->me_maxdbs * sizeof(MDB_cursor *);
-		/* child txns use parent's dbiseqs */
-		if (!parent)
-			size += env->me_maxdbs * sizeof(unsigned int);
 	}
+	size += env->me_maxdbs * (sizeof(MDB_db)+1);
 
 	if ((txn = calloc(1, size)) == NULL) {
 		DPRINTF(("calloc: %s", strerror(errno)));
@@ -2807,38 +2847,40 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
 		txn->mt_numdbs = 0;		/* close nothing if called again */
 		txn->mt_dbxs = NULL;	/* mark txn as reset */
 	} else {
-		mdb_cursors_close(txn, 0);
+		pgno_t *pghead = env->me_pghead;
 
+		mdb_cursors_close(txn, 0);
 		if (!(env->me_flags & MDB_WRITEMAP)) {
 			mdb_dlist_free(txn);
 		}
-		mdb_midl_free(env->me_pghead);
 
-		if (txn->mt_parent) {
+		if (!txn->mt_parent) {
+			if (mdb_midl_shrink(&txn->mt_free_pgs))
+				env->me_free_pgs = txn->mt_free_pgs;
+			/* me_pgstate: */
+			env->me_pghead = NULL;
+			env->me_pglast = 0;
+
+			env->me_txn = NULL;
+			/* The writer mutex was locked in mdb_txn_begin. */
+			if (env->me_txns)
+				UNLOCK_MUTEX_W(env);
+		} else {
 			txn->mt_parent->mt_child = NULL;
 			env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
 			mdb_midl_free(txn->mt_free_pgs);
 			mdb_midl_free(txn->mt_spill_pgs);
 			free(txn->mt_u.dirty_list);
-			return;
 		}
 
-		if (mdb_midl_shrink(&txn->mt_free_pgs))
+		mdb_midl_free(pghead);
-			env->me_free_pgs = txn->mt_free_pgs;
-		env->me_pghead = NULL;
-		env->me_pglast = 0;
-
-		env->me_txn = NULL;
-		/* The writer mutex was locked in mdb_txn_begin. */
-		if (env->me_txns)
-			UNLOCK_MUTEX_W(env);
 	}
 #ifdef VL32
 	{
 		unsigned i, n = txn->mt_rpages[0].mid;
 		for (i = 1; i <= n; i++) {
 #ifdef _WIN32
-			UnmapViewOfFile(txn->mt_rpages[i].mptr);)
+			UnmapViewOfFile(txn->mt_rpages[i].mptr);
 #else
 			MDB_page *mp = txn->mt_rpages[i].mptr;
 			int size = txn->mt_env->me_psize;
@@ -3159,6 +3201,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
 		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
 			if (n) {
+retry_write:
 				/* Write previous page(s) */
 #ifdef MDB_USE_PWRITEV
 				wres = pwritev(env->me_fd, iov, n, wpos);
@@ -3166,8 +3209,11 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				if (n == 1) {
 					wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
 				} else {
+retry_seek:
 					if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
 						rc = ErrCode();
+						if (rc == EINTR)
+							goto retry_seek;
 						DPRINTF(("lseek: %s", strerror(rc)));
 						return rc;
 					}
@@ -3177,6 +3223,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				if (wres != wsize) {
 					if (wres < 0) {
 						rc = ErrCode();
+						if (rc == EINTR)
+							goto retry_write;
 						DPRINTF(("Write error: %s", strerror(rc)));
 					} else {
 						rc = EIO; /* TODO: Use which error code? */
@@ -3546,7 +3594,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	int len;
 #define DO_PWRITE(rc, fd, ptr, size, len, pos)	do { \
 	len = pwrite(fd, ptr, size, pos);	\
-	rc = (len >= 0); } while(0)
+	if (len == -1 && ErrCode() == EINTR) continue; \
+	rc = (len >= 0); break; } while(1)
 #endif
 
 	DPUTS("writing new meta page");
@@ -3651,6 +3700,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	/* Write to the SYNC fd */
 	mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ?
 		env->me_fd : env->me_mfd;
+retry_write:
 #ifdef _WIN32
 	{
 		memset(&ov, 0, sizeof(ov));
@@ -3663,6 +3713,8 @@ mdb_env_write_meta(MDB_txn *txn)
 #endif
 	if (rc != len) {
 		rc = rc < 0 ? ErrCode() : EIO;
+		if (rc == EINTR)
+			goto retry_write;
 		DPUTS("write failed, disk error?");
 		/* On a failure, the pagecache still contains the new data.
 		 * Write some old data back, to prevent it from being used.
@@ -3894,6 +3946,32 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers)
 	return MDB_SUCCESS;
 }
 
+static int ESECT
+mdb_fsize(HANDLE fd, size_t *size)
+{
+#ifdef _WIN32
+	LARGE_INTEGER fsize;
+
+	if (!GetFileSizeEx(fd, &fsize))
+		return ErrCode();
+
+	*size = fsize.QuadPart;
+#else
+	struct stat st;
+
+	if (fstat(fd, &st))
+		return ErrCode();
+
+	*size = st.st_size;
+#endif
+	return MDB_SUCCESS;
+}
+
+#ifdef BROKEN_FDATASYNC
+#include <sys/utsname.h>
+#include <sys/vfs.h>
+#endif
+
 /** Further setup required for opening an LMDB environment
  */
 static int ESECT
@@ -3911,6 +3989,53 @@ mdb_env_open2(MDB_env *env)
 	else
 		env->me_pidquery = PROCESS_QUERY_INFORMATION;
 #endif /* _WIN32 */
+#ifdef BROKEN_FDATASYNC
+	/* ext3/ext4 fdatasync is broken on some older Linux kernels.
+	 * https://lkml.org/lkml/2012/9/3/83
+	 * Kernels after 3.6-rc6 are known good.
+	 * https://lkml.org/lkml/2012/9/10/556
+	 * See if the DB is on ext3/ext4, then check for new enough kernel
+	 * Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known
+	 * to be patched.
+	 */
+	{
+		struct statfs st;
+		fstatfs(env->me_fd, &st);
+		while (st.f_type == 0xEF53) {
+			struct utsname uts;
+			int i;
+			uname(&uts);
+			if (uts.release[0] < '3') {
+				if (!strncmp(uts.release, "2.6.32.", 7)) {
+					i = atoi(uts.release+7);
+					if (i >= 60)
+						break;	/* 2.6.32.60 and newer is OK */
+				} else if (!strncmp(uts.release, "2.6.34.", 7)) {
+					i = atoi(uts.release+7);
+					if (i >= 15)
+						break;	/* 2.6.34.15 and newer is OK */
+				}
+			} else if (uts.release[0] == '3') {
+				i = atoi(uts.release+2);
+				if (i > 5)
+					break;	/* 3.6 and newer is OK */
+				if (i == 5) {
+					i = atoi(uts.release+4);
+					if (i >= 4)
+						break;	/* 3.5.4 and newer is OK */
+				} else if (i == 2) {
+					i = atoi(uts.release+4);
+					if (i >= 30)
+						break;	/* 3.2.30 and newer is OK */
+				}
+			} else {	/* 4.x and newer is OK */
+				break;
+			}
+			env->me_flags |= MDB_FSYNCONLY;
+			break;
+		}
+	}
+#endif
 
 	memset(&meta, 0, sizeof(meta));
 
@@ -4042,7 +4167,7 @@ PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_
 extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp;
 const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback;
 #pragma const_seg()
-#else	/* WIN32 */
+#else	/* _WIN32 */
 #pragma comment(linker, "/INCLUDE:__tls_used")
 #pragma comment(linker, "/INCLUDE:_mdb_tls_cbp")
 #pragma data_seg(".CRT$XLB")
@@ -4092,7 +4217,7 @@ mdb_env_share_locks(MDB_env *env, int *excl)
 	return rc;
 }
 
-/** Try to get exlusive lock, otherwise shared.
+/** Try to get exclusive lock, otherwise shared.
  *	Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive.
  */
 static int ESECT
@@ -4233,7 +4358,6 @@ mdb_hash_enc(MDB_val *val, char *encbuf)
  * @param[in] env The LMDB environment.
  * @param[in] lpath The pathname of the file used for the lock region.
  * @param[in] mode The Unix permissions for the file, if we create it.
- * @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive
  * @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive
  * @return 0 on success, non-zero on failure.
  */
@@ -4599,7 +4723,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 		if (!(flags & MDB_RDONLY)) {
 			MDB_txn *txn;
 			int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs *
-				(sizeof(MDB_db)+sizeof(MDB_cursor)+sizeof(unsigned int)+1);
+				(sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1);
 			txn = calloc(1, size);
 			if (txn) {
 				txn->mt_dbs = (MDB_db *)((char *)txn + tsize);
@@ -4607,6 +4731,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
 				txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
 				txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
 				txn->mt_env = env;
+				txn->mt_dbxs = env->me_dbxs;
 #ifdef VL32
 				txn->mt_rpages = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2));
 				if (!txn->mt_rpages) {
@@ -4639,13 +4764,15 @@ mdb_env_close0(MDB_env *env, int excl)
 		return;
 
 	/* Doing this here since me_dbxs may not exist during mdb_env_close */
-	for (i = env->me_maxdbs; --i > MAIN_DBI; )
+	if (env->me_dbxs) {
-		free(env->me_dbxs[i].md_name.mv_data);
+		for (i = env->me_maxdbs; --i > MAIN_DBI; )
+			free(env->me_dbxs[i].md_name.mv_data);
+		free(env->me_dbxs);
+	}
 
 	free(env->me_pbuf);
 	free(env->me_dbiseqs);
 	free(env->me_dbflags);
-	free(env->me_dbxs);
 	free(env->me_path);
 	free(env->me_dirty_list);
 	free(env->me_txn0);
@@ -4752,7 +4879,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 		*(size_t *)a->mv_data > *(size_t *)b->mv_data;
 }
 
-/** Compare two items pointing at aligned unsigned int's */
+/** Compare two items pointing at aligned unsigned int's.
+ *
+ *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
+ *	but #mdb_cmp_clong() is called instead if the data type is size_t.
+ */
 static int
 mdb_cmp_int(const MDB_val *a, const MDB_val *b)
 {
@@ -4790,13 +4921,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
 #endif
 }
 
-/** Compare two items pointing at size_t's of unknown alignment. */
-#ifdef MISALIGNED_OK
-# define mdb_cmp_clong mdb_cmp_long
-#else
-# define mdb_cmp_clong mdb_cmp_cint
-#endif
-
 /** Compare two items lexically */
 static int
 mdb_cmp_memn(const MDB_val *a, const MDB_val *b)
@@ -5432,6 +5556,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
 #ifdef VL32
 	op = mc->mc_pg[mc->mc_top];
 #endif
+
 	mdb_cursor_pop(mc);
 	DPRINTF(("parent page is page %"Z"u, index %u",
 		mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]));
@@ -5590,11 +5715,11 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
 					}
 					return rc;
 				}
-			} else {
-				mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
-				if (op == MDB_PREV_DUP)
-					return MDB_NOTFOUND;
 			}
+		} else {
+			mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
+			if (op == MDB_PREV_DUP)
+				return MDB_NOTFOUND;
 		}
 	}
 
@@ -5806,15 +5931,21 @@ set1:
 					return rc;
 			}
 		} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
-			MDB_val d2;
+			MDB_val olddata;
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS)
+			MDB_cmp_func *dcmp;
+			if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
 				return rc;
-			rc = mc->mc_dbx->md_dcmp(data, &d2);
+			dcmp = mc->mc_dbx->md_dcmp;
+#if UINT_MAX < SIZE_MAX
+			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+				dcmp = mdb_cmp_clong;
+#endif
+			rc = dcmp(data, &olddata);
 			if (rc) {
 				if (op == MDB_GET_BOTH || rc > 0)
 					return MDB_NOTFOUND;
 				rc = 0;
-				*data = d2;
+				*data = olddata;
 			}
 
 		} else {
@@ -6324,16 +6455,17 @@ more:
 
 			/* Was a single item before, must convert now */
 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
+				MDB_cmp_func *dcmp;
 				/* Just overwrite the current item */
 				if (flags == MDB_CURRENT)
 					goto current;
-
+				dcmp = mc->mc_dbx->md_dcmp;
 #if UINT_MAX < SIZE_MAX
-				if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
-					mc->mc_dbx->md_dcmp = mdb_cmp_clong;
+					dcmp = mdb_cmp_clong;
 #endif
 				/* does data match? */
-				if (!mc->mc_dbx->md_dcmp(data, &olddata)) {
+				if (!dcmp(data, &olddata)) {
 					if (flags & MDB_NODUPDATA)
 						return MDB_KEYEXIST;
 					/* overwrite it */
@@ -7153,6 +7285,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
 	mc->mc_snum = 0;
 	mc->mc_top = 0;
 	mc->mc_pg[0] = 0;
+	mc->mc_ki[0] = 0;
 	mc->mc_flags = 0;
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_tassert(txn, mx != NULL);
@@ -7526,7 +7659,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
 			cdst->mc_ki[cdst->mc_top] = 0;
 			rc = mdb_update_key(cdst, &nullkey);
 			cdst->mc_ki[cdst->mc_top] = ix;
-			mdb_cassert(csrc, rc == MDB_SUCCESS);
+			mdb_cassert(cdst, rc == MDB_SUCCESS);
 		}
 	}
 
@@ -7782,12 +7915,12 @@ mdb_rebalance(MDB_cursor *mc)
 						m3 = m2;
 					if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
 					if (m3->mc_pg[0] == mp) {
-						m3->mc_snum--;
-						m3->mc_top--;
 						for (i=0; i<m3->mc_snum; i++) {
 							m3->mc_pg[i] = m3->mc_pg[i+1];
 							m3->mc_ki[i] = m3->mc_ki[i+1];
 						}
+						m3->mc_snum--;
+						m3->mc_top--;
 					}
 				}
 			}
@@ -7855,9 +7988,23 @@ mdb_rebalance(MDB_cursor *mc)
 		if (mc->mc_ki[ptop] == 0) {
 			rc = mdb_page_merge(&mn, mc);
 		} else {
+			MDB_cursor dummy;
 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
+			/* We want mdb_rebalance to find mn when doing fixups */
+			if (mc->mc_flags & C_SUB) {
+				dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
+				dummy.mc_xcursor = (MDB_xcursor *)&mn;
+			} else {
+				mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
+			}
 			rc = mdb_page_merge(mc, &mn);
+			if (mc->mc_flags & C_SUB)
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next;
+			else
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
 			mdb_cursor_copy(&mn, mc);
 		}
 		mc->mc_flags &= ~C_EOF;
@@ -7884,6 +8031,13 @@ mdb_cursor_del0(MDB_cursor *mc)
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = mc->mc_dbi;
 
+		/* DB is totally empty now, just bail out.
+		 * Other cursors adjustments were already done
+		 * by mdb_rebalance and aren't needed here.
+		 */
+		if (!mc->mc_snum)
+			return rc;
+
 		mp = mc->mc_pg[mc->mc_top];
 		nkeys = NUMKEYS(mp);
 
@@ -8751,8 +8905,12 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
 		/* Set metapage 1 */
 		mm->mm_last_pg = txn->mt_next_pgno - freecount - 1;
 		mm->mm_dbs[1] = txn->mt_dbs[1];
-		mm->mm_dbs[1].md_root = mm->mm_last_pg;
+		if (mm->mm_last_pg > 1) {
-		mm->mm_txnid = 1;
+			mm->mm_dbs[1].md_root = mm->mm_last_pg;
+			mm->mm_txnid = 1;
+		} else {
+			mm->mm_dbs[1].md_root = P_INVALID;
+		}
 	}
 	my.mc_wlen[0] = env->me_psize * 2;
 	my.mc_txn = txn;
@@ -8847,21 +9005,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
 		goto leave;
 
 	w2 = txn->mt_next_pgno * env->me_psize;
-#ifdef WIN32
 	{
-		LARGE_INTEGER fsize;
+		size_t fsize = 0;
-		GetFileSizeEx(env->me_fd, &fsize);
+		if ((rc = mdb_fsize(env->me_fd, &fsize)))
-		if (w2 > fsize.QuadPart)
+			goto leave;
-			w2 = fsize.QuadPart;
+		if (w2 > fsize)
+			w2 = fsize;
 	}
-#else
-	{
-		struct stat st;
-		fstat(env->me_fd, &st);
-		if (w2 > (size_t)st.st_size)
-			w2 = st.st_size;
-	}
-#endif
 	wsize = w2 - wsize;
 	while (wsize > 0) {
 		if (wsize > MAX_WRITE)

external/db_drivers/liblmdb32/mdb_copy.1

@@ -1,5 +1,5 @@
 .TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_copy \- LMDB environment copy tool

external/db_drivers/liblmdb32/mdb_copy.c

@@ -1,6 +1,6 @@
 /* mdb_copy.c - memory-mapped database backup tool */
 /*
- * Copyright 2012 Howard Chu, Symas Corp.
+ * Copyright 2012-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -64,7 +64,7 @@ int main(int argc,char * argv[])
 	act = "opening environment";
 	rc = mdb_env_create(&env);
 	if (rc == MDB_SUCCESS) {
-		rc = mdb_env_open(env, argv[1], flags, 0664);
+		rc = mdb_env_open(env, argv[1], flags, 0600);
 	}
 	if (rc == MDB_SUCCESS) {
 		act = "copying";

external/db_drivers/liblmdb32/mdb_dump.1

@@ -1,5 +1,5 @@
 .TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_dump \- LMDB environment export tool

external/db_drivers/liblmdb32/mdb_dump.c

@@ -1,6 +1,6 @@
 /* mdb_dump.c - memory-mapped database dump tool */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

external/db_drivers/liblmdb32/mdb_load.1

@@ -1,5 +1,5 @@
 .TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_load \- LMDB environment import tool

external/db_drivers/liblmdb32/mdb_load.c

@@ -1,6 +1,6 @@
 /* mdb_load.c - memory-mapped database load tool */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -176,7 +176,7 @@ static int unhex(unsigned char *c2)
 static int readline(MDB_val *out, MDB_val *buf)
 {
 	unsigned char *c1, *c2, *end;
-	size_t len;
+	size_t len, l2;
 	int c;
 
 	if (!(mode & NOHDR)) {
@@ -206,6 +206,7 @@ badend:
 
 	c1 = buf->mv_data;
 	len = strlen((char *)c1);
+	l2 = len;
 
 	/* Is buffer too short? */
 	while (c1[len-1] != '\n') {
@@ -217,17 +218,18 @@ badend:
 			return EOF;
 		}
 		c1 = buf->mv_data;
-		c1 += buf->mv_size;
+		c1 += l2;
-		if (fgets((char *)c1, buf->mv_size, stdin) == NULL) {
+		if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
 			Eof = 1;
 			badend();
 			return EOF;
 		}
 		buf->mv_size *= 2;
 		len = strlen((char *)c1);
+		l2 += len;
 	}
 	c1 = c2 = buf->mv_data;
-	len = strlen((char *)c1);
+	len = l2;
 	c1[--len] = '\0';
 	end = c1 + len;
 

external/db_drivers/liblmdb32/mdb_stat.1

@@ -1,5 +1,5 @@
 .TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
-.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
+.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
 .\" Copying restrictions apply.  See COPYRIGHT/LICENSE.
 .SH NAME
 mdb_stat \- LMDB environment status tool

external/db_drivers/liblmdb32/mdb_stat.c

@@ -1,6 +1,6 @@
 /* mdb_stat.c - memory-mapped database status tool */
 /*
- * Copyright 2011-2013 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

external/db_drivers/liblmdb32/midl.c

@@ -3,7 +3,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2014 The OpenLDAP Foundation.
+ * Copyright 2000-2015 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

external/db_drivers/liblmdb32/midl.h

@@ -11,7 +11,7 @@
 /* $OpenLDAP$ */
 /* This work is part of OpenLDAP Software <http://www.openldap.org/>.
  *
- * Copyright 2000-2014 The OpenLDAP Foundation.
+ * Copyright 2000-2015 The OpenLDAP Foundation.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -58,7 +58,7 @@ typedef MDB_ID *MDB_IDL;
 #ifdef VL32
 #define	MDB_IDL_LOGN	10	/* DB_SIZE is 2^10, UM_SIZE is 2^11 */
 #else
-#define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
+ #define	MDB_IDL_LOGN	16	/* DB_SIZE is 2^16, UM_SIZE is 2^17 */
 #endif
 #define MDB_IDL_DB_SIZE		(1<<MDB_IDL_LOGN)
 #define MDB_IDL_UM_SIZE		(1<<(MDB_IDL_LOGN+1))

external/db_drivers/liblmdb32/mtest.c

@@ -1,6 +1,6 @@
 /* mtest.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -45,19 +45,22 @@ int main(int argc,char * argv[])
 	    }
     
 		E(mdb_env_create(&env));
+		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
 		E(mdb_env_open(env, "./testdb", 0 /* MDB_FIXEDMAP |MDB_NOSYNC*/, 0664));
+
 		E(mdb_txn_begin(env, NULL, 0, &txn));
-		E(mdb_open(txn, NULL, 0, &dbi));
+		E(mdb_dbi_open(txn, NULL, 0, &dbi));
    
 		key.mv_size = sizeof(int);
 		key.mv_data = sval;
-		data.mv_size = sizeof(sval);
-		data.mv_data = sval;
 
 		printf("Adding %d values\n", count);
 	    for (i=0;i<count;i++) {	
 			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
 			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
 				j++;
 				data.mv_size = sizeof(sval);
@@ -68,7 +71,7 @@ int main(int argc,char * argv[])
 		E(mdb_txn_commit(txn));
 		E(mdb_env_stat(env, &mst));
 
-		E(mdb_txn_begin(env, NULL, 1, &txn));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 		E(mdb_cursor_open(txn, dbi, &cursor));
 		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 			printf("key: %p %.*s, data: %p %.*s\n",
@@ -97,7 +100,7 @@ int main(int argc,char * argv[])
 		printf("Deleted %d values\n", j);
 
 		E(mdb_env_stat(env, &mst));
-		E(mdb_txn_begin(env, NULL, 1, &txn));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 		E(mdb_cursor_open(txn, dbi, &cursor));
 		printf("Cursor next\n");
 		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -128,6 +131,7 @@ int main(int argc,char * argv[])
 				(int) key.mv_size,  (char *) key.mv_data,
 				(int) data.mv_size, (char *) data.mv_data);
 
+		mdb_cursor_close(cursor);
 		mdb_txn_abort(txn);
 
 		printf("Deleting with cursor\n");
@@ -164,9 +168,9 @@ int main(int argc,char * argv[])
 				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
 		}
 		mdb_cursor_close(cursor);
-		mdb_close(env, dbi);
-
 		mdb_txn_abort(txn);
+
+		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
 
 	return 0;

external/db_drivers/liblmdb32/mtest2.c

@@ -1,6 +1,6 @@
 /* mtest2.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -47,20 +47,22 @@ int main(int argc,char * argv[])
 	}
 
 	E(mdb_env_create(&env));
+	E(mdb_env_set_maxreaders(env, 1));
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
+	E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
    
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
-	data.mv_size = sizeof(sval);
-	data.mv_data = sval;
 
 	printf("Adding %d values\n", count);
 	for (i=0;i<count;i++) {	
 		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+		data.mv_size = sizeof(sval);
+		data.mv_data = sval;
 		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
 			j++;
 	}
@@ -68,7 +70,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -97,7 +99,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -114,10 +116,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb32/mtest3.c

@@ -1,6 +1,6 @@
 /* mtest3.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -53,8 +53,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
+	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
 
 	key.mv_size = sizeof(int);
 	key.mv_data = kval;
@@ -73,7 +74,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -107,7 +108,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -124,10 +125,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb32/mtest4.c

@@ -1,6 +1,6 @@
 /* mtest4.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -51,8 +51,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
+	E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
 
 	key.mv_size = sizeof(int);
 	key.mv_data = kval;
@@ -72,7 +73,7 @@ int main(int argc,char * argv[])
 
 	/* there should be one full page of dups now.
 	 */
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -142,7 +143,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -159,10 +160,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb32/mtest5.c

@@ -1,6 +1,6 @@
 /* mtest5.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -53,8 +53,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
+	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 
 	key.mv_size = sizeof(int);
@@ -75,7 +76,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -109,7 +110,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -126,10 +127,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb32/mtest6.c

@@ -1,6 +1,6 @@
 /* mtest6.c - memory-mapped database tester/toy */
 /*
- * Copyright 2011-2014 Howard Chu, Symas Corp.
+ * Copyright 2011-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,7 @@ int main(int argc,char * argv[])
 	int i = 0, j = 0, rc;
 	MDB_env *env;
 	MDB_dbi dbi;
-	MDB_val key, data;
+	MDB_val key, data, sdata;
 	MDB_txn *txn;
 	MDB_stat mst;
 	MDB_cursor *cursor;
@@ -46,33 +46,37 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
+	E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	E(mdb_stat(txn, dbi, &mst));
 
 	sval = calloc(1, mst.ms_psize / 4);
 	key.mv_size = sizeof(long);
 	key.mv_data = &kval;
-	data.mv_size = mst.ms_psize / 4 - 30;
+	sdata.mv_size = mst.ms_psize / 4 - 30;
-	data.mv_data = sval;
+	sdata.mv_data = sval;
 
 	printf("Adding 12 values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+4;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+1;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
@@ -110,7 +114,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -127,9 +131,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
+
+	mdb_dbi_close(env, dbi);
 #endif
 	mdb_env_close(env);
 

external/db_drivers/liblmdb32/sample-bdb.txt

@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-mdb.txt
  */
 /*
- * Copyright 2012 Howard Chu, Symas Corp.
+ * Copyright 2012-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without

external/db_drivers/liblmdb32/sample-mdb.txt

@@ -3,7 +3,7 @@
  * Do a line-by-line comparison of this and sample-bdb.txt
  */
 /*
- * Copyright 2012 Howard Chu, Symas Corp.
+ * Copyright 2012-2015 Howard Chu, Symas Corp.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -32,7 +32,7 @@ int main(int argc,char * argv[])
 	rc = mdb_env_create(&env);
 	rc = mdb_env_open(env, "./testdb", 0, 0664);
 	rc = mdb_txn_begin(env, NULL, 0, &txn);
-	rc = mdb_open(txn, NULL, 0, &dbi);
+	rc = mdb_dbi_open(txn, NULL, 0, &dbi);
 
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
@@ -56,7 +56,7 @@ int main(int argc,char * argv[])
 	mdb_cursor_close(cursor);
 	mdb_txn_abort(txn);
 leave:
-	mdb_close(env, dbi);
+	mdb_dbi_close(env, dbi);
 	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb64/CHANGES

@@ -1,18 +1,31 @@
 LMDB 0.9 Change Log
 
-LMDB 0.9.15 Release Engineering
+LMDB 0.9.15 Release (2015/06/19)
 	Fix txn init (ITS#7961,#7987)
 	Fix MDB_PREV_DUP (ITS#7955,#7671)
 	Fix compact of empty env (ITS#7956)
+	Fix mdb_copy file mode
+	Fix mdb_env_close() after failed mdb_env_open()
+	Fix mdb_rebalance collapsing root (ITS#8062)
 	Fix mdb_load with large values (ITS#8066)
+	Fix to retry writes on EINTR (ITS#8106)
+	Fix mdb_cursor_del on empty DB (ITS#8109)
+	Fix MDB_INTEGERDUP key compare (ITS#8117)
+	Fix error handling (ITS#7959,#8157,etc.)
+	Fix race conditions (ITS#7969,7970)
 	Added workaround for fdatasync bug in ext3fs
 	Build
 		Don't use -fPIC for static lib
 		Update .gitignore (ITS#7952,#7953)
-		Cleanup for "make test" (ITS#7841)
+		Cleanup for "make test" (ITS#7841), "make clean", mtest*.c
 		Misc. Android/Windows cleanup
 	Documentation
 		Fix MDB_APPEND doc
+		Fix MDB_MAXKEYSIZE doc (ITS#8156)
+		Fix mdb_cursor_put,mdb_cursor_del EACCES description
+		Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021)
+		Clarify MDB_WRITEMAP doc (ITS#8021)
+		Clarify mdb_env_open doc
 		Clarify mdb_dbi_open doc
 
 LMDB 0.9.14 Release (2014/09/20)

external/db_drivers/liblmdb64/CMakeLists.txt

@@ -26,6 +26,10 @@
 # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
 # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+if(FREEBSD)
+  set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMDB_DSYNC=O_SYNC")
+endif()
+
 set (lmdb_sources
 mdb.c
 midl.c)

external/db_drivers/liblmdb64/lmdb.h

@@ -184,7 +184,7 @@ typedef int mdb_filehandle_t;
 /** Library minor version */
 #define MDB_VERSION_MINOR	9
 /** Library patch version */
-#define MDB_VERSION_PATCH	14
+#define MDB_VERSION_PATCH	15
 
 /** Combine args a,b,c into a single integer for easy version comparisons */
 #define MDB_VERINT(a,b,c)	(((a) << 24) | ((b) << 16) | (c))
@@ -194,7 +194,7 @@ typedef int mdb_filehandle_t;
 	MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
 
 /** The release date of this library version */
-#define MDB_VERSION_DATE	"September 20, 2014"
+#define MDB_VERSION_DATE	"June 19, 2015"
 
 /** A stringifier for the version info */
 #define MDB_VERSTR(a,b,c,d)	"LMDB " #a "." #b "." #c ": (" d ")"
@@ -296,12 +296,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
 #define MDB_REVERSEKEY	0x02
 	/** use sorted duplicates */
 #define MDB_DUPSORT		0x04
-	/** numeric keys in native byte order.
+	/** numeric keys in native byte order: either unsigned int or size_t.
 	 *  The keys must all be of the same size. */
 #define MDB_INTEGERKEY	0x08
 	/** with #MDB_DUPSORT, sorted dup items have fixed size */
 #define MDB_DUPFIXED	0x10
-	/** with #MDB_DUPSORT, dups are numeric in native byte order */
+	/** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
 #define MDB_INTEGERDUP	0x20
 	/** with #MDB_DUPSORT, use reverse string dups */
 #define MDB_REVERSEDUP	0x40
@@ -588,8 +588,8 @@ int  mdb_env_create(MDB_env **env);
 	 *		reserved in that case.
 	 *		This flag may be changed at any time using #mdb_env_set_flags().
 	 * </ul>
-	 * @param[in] mode The UNIX permissions to set on created files. This parameter
+	 * @param[in] mode The UNIX permissions to set on created files and semaphores.
-	 * is ignored on Windows.
+	 * This parameter is ignored on Windows.
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
@@ -1021,14 +1021,16 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 * The database handle may be discarded by calling #mdb_dbi_close().
 	 * The old database handle is returned if the database was already open.
 	 * The handle may only be closed once.
+	 *
 	 * The database handle will be private to the current transaction until
 	 * the transaction is successfully committed. If the transaction is
 	 * aborted the handle will be closed automatically.
-	 * After a successful commit the
+	 * After a successful commit the handle will reside in the shared
-	 * handle will reside in the shared environment, and may be used
+	 * environment, and may be used by other transactions.
-	 * by other transactions. This function must not be called from
+	 *
-	 * multiple concurrent transactions in the same process. A transaction
+	 * This function must not be called from multiple concurrent
-	 * that uses this function must finish (either commit or abort) before
+	 * transactions in the same process. A transaction that uses
+	 * this function must finish (either commit or abort) before
 	 * any other transaction in the process may use this function.
 	 *
 	 * To use named databases (with name != NULL), #mdb_env_set_maxdbs()
@@ -1050,9 +1052,9 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		keys may have multiple data items, stored in sorted order.) By default
 	 *		keys must be unique and may have only a single data item.
 	 *	<li>#MDB_INTEGERKEY
-	 *		Keys are binary integers in native byte order. Setting this option
+	 *		Keys are binary integers in native byte order, either unsigned int
-	 *		requires all keys to be the same size, typically sizeof(int)
+	 *		or size_t, and will be sorted as such.
-	 *		or sizeof(size_t).
+	 *		The keys must all be of the same size.
 	 *	<li>#MDB_DUPFIXED
 	 *		This flag may only be used in combination with #MDB_DUPSORT. This option
 	 *		tells the library that the data items for this database are all the same
@@ -1060,8 +1062,8 @@ int  mdb_txn_renew(MDB_txn *txn);
 	 *		all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
 	 *		cursor operations may be used to retrieve multiple items at once.
 	 *	<li>#MDB_INTEGERDUP
-	 *		This option specifies that duplicate data items are also integers, and
+	 *		This option specifies that duplicate data items are binary integers,
-	 *		should be sorted as such.
+	 *		similar to #MDB_INTEGERKEY keys.
 	 *	<li>#MDB_REVERSEDUP
 	 *		This option specifies that duplicate data items should be compared as
 	 *		strings in reverse order.
@@ -1450,7 +1452,7 @@ int  mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 * <ul>
 	 *	<li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
 	 *	<li>#MDB_TXN_FULL - the transaction has too many dirty pages.
-	 *	<li>EACCES - an attempt was made to modify a read-only database.
+	 *	<li>EACCES - an attempt was made to write in a read-only transaction.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 * </ul>
 	 */
@@ -1470,7 +1472,7 @@ int  mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
 	 * @return A non-zero error value on failure and 0 on success. Some possible
 	 * errors are:
 	 * <ul>
-	 *	<li>EACCES - an attempt was made to modify a read-only database.
+	 *	<li>EACCES - an attempt was made to write in a read-only transaction.
 	 *	<li>EINVAL - an invalid parameter was specified.
 	 * </ul>
 	 */

external/db_drivers/liblmdb64/mdb.c

@@ -446,12 +446,17 @@ static txnid_t mdb_debug_start;
 	/**	The version number for a database's lockfile format. */
 #define MDB_LOCK_VERSION	 1
 
-	/**	@brief The max size of a key we can write, or 0 for dynamic max.
+	/**	@brief The max size of a key we can write, or 0 for computed max.
 	 *
-	 *	Define this as 0 to compute the max from the page size.  511
+	 *	This macro should normally be left alone or set to 0.
-	 *	is default for backwards compat: liblmdb <= 0.9.10 can break
+	 *	Note that a database with big keys or dupsort data cannot be
-	 *	when modifying a DB with keys/dupsort data bigger than its max.
+	 *	reliably modified by a liblmdb which uses a smaller max.
-	 *	#MDB_DEVEL sets the default to 0.
+	 *	The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
+	 *
+	 *	Other values are allowed, for backwards compat.  However:
+	 *	A value bigger than the computed max can break if you do not
+	 *	know what you are doing, and liblmdb <= 0.9.10 can break when
+	 *	modifying a DB with keys/dupsort data bigger than its max.
 	 *
 	 *	Data items in an #MDB_DUPSORT database are also limited to
 	 *	this size, since they're actually keys of a sub-DB.  Keys and
@@ -1245,6 +1250,13 @@ static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
 static MDB_cmp_func	mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
 /** @endcond */
 
+/** Compare two items pointing at size_t's of unknown alignment. */
+#ifdef MISALIGNED_OK
+# define mdb_cmp_clong mdb_cmp_long
+#else
+# define mdb_cmp_clong mdb_cmp_cint
+#endif
+
 #ifdef _WIN32
 static SECURITY_DESCRIPTOR mdb_null_sd;
 static SECURITY_ATTRIBUTES mdb_all_sa;
@@ -1558,7 +1570,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 int
 mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
 {
-	return txn->mt_dbxs[dbi].md_dcmp(a, b);
+	MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
+#if UINT_MAX < SIZE_MAX
+	if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
+		dcmp = mdb_cmp_clong;
+#endif
+	return dcmp(a, b);
 }
 
 /** Allocate memory for a page.
@@ -2486,14 +2503,11 @@ mdb_txn_renew0(MDB_txn *txn)
 	MDB_env *env = txn->mt_env;
 	MDB_txninfo *ti = env->me_txns;
 	MDB_meta *meta;
-	unsigned int i, nr;
+	unsigned int i, nr, flags = txn->mt_flags;
 	uint16_t x;
 	int rc, new_notls = 0;
 
-	if (txn->mt_flags & MDB_TXN_RDONLY) {
+	if ((flags &= MDB_TXN_RDONLY) != 0) {
-		/* Setup db info */
-		txn->mt_numdbs = env->me_numdbs;
-		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
 		if (!ti) {
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
@@ -2546,6 +2560,7 @@ mdb_txn_renew0(MDB_txn *txn)
 			txn->mt_u.reader = r;
 			meta = env->me_metas[txn->mt_txnid & 1];
 		}
+		txn->mt_dbxs = env->me_dbxs;	/* mostly static anyway */
 	} else {
 		if (ti) {
 			LOCK_MUTEX_W(env);
@@ -2556,14 +2571,11 @@ mdb_txn_renew0(MDB_txn *txn)
 			meta = env->me_metas[ mdb_env_pick_meta(env) ];
 			txn->mt_txnid = meta->mm_txnid;
 		}
-		/* Setup db info */
-		txn->mt_numdbs = env->me_numdbs;
 		txn->mt_txnid++;
 #if MDB_DEBUG
 		if (txn->mt_txnid == mdb_debug_start)
 			mdb_debug = 1;
 #endif
-		txn->mt_flags = 0;
 		txn->mt_child = NULL;
 		txn->mt_loose_pgs = NULL;
 		txn->mt_loose_count = 0;
@@ -2583,6 +2595,10 @@ mdb_txn_renew0(MDB_txn *txn)
 	/* Moved to here to avoid a data race in read TXNs */
 	txn->mt_next_pgno = meta->mm_last_pg+1;
 
+	txn->mt_flags = flags;
+
+	/* Setup db info */
+	txn->mt_numdbs = env->me_numdbs;
 	for (i=2; i<txn->mt_numdbs; i++) {
 		x = env->me_dbflags[i];
 		txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
@@ -3132,6 +3148,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
 		/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
 		if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
 			if (n) {
+retry_write:
 				/* Write previous page(s) */
 #ifdef MDB_USE_PWRITEV
 				wres = pwritev(env->me_fd, iov, n, wpos);
@@ -3139,8 +3156,11 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				if (n == 1) {
 					wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
 				} else {
+retry_seek:
 					if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
 						rc = ErrCode();
+						if (rc == EINTR)
+							goto retry_seek;
 						DPRINTF(("lseek: %s", strerror(rc)));
 						return rc;
 					}
@@ -3150,6 +3170,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
 				if (wres != wsize) {
 					if (wres < 0) {
 						rc = ErrCode();
+						if (rc == EINTR)
+							goto retry_write;
 						DPRINTF(("Write error: %s", strerror(rc)));
 					} else {
 						rc = EIO; /* TODO: Use which error code? */
@@ -3519,7 +3541,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
 	int len;
 #define DO_PWRITE(rc, fd, ptr, size, len, pos)	do { \
 	len = pwrite(fd, ptr, size, pos);	\
-	rc = (len >= 0); } while(0)
+	if (len == -1 && ErrCode() == EINTR) continue; \
+	rc = (len >= 0); break; } while(1)
 #endif
 
 	DPUTS("writing new meta page");
@@ -3624,6 +3647,7 @@ mdb_env_write_meta(MDB_txn *txn)
 	/* Write to the SYNC fd */
 	mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ?
 		env->me_fd : env->me_mfd;
+retry_write:
 #ifdef _WIN32
 	{
 		memset(&ov, 0, sizeof(ov));
@@ -3636,6 +3660,8 @@ mdb_env_write_meta(MDB_txn *txn)
 #endif
 	if (rc != len) {
 		rc = rc < 0 ? ErrCode() : EIO;
+		if (rc == EINTR)
+			goto retry_write;
 		DPUTS("write failed, disk error?");
 		/* On a failure, the pagecache still contains the new data.
 		 * Write some old data back, to prevent it from being used.
@@ -4761,7 +4787,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
 		*(size_t *)a->mv_data > *(size_t *)b->mv_data;
 }
 
-/** Compare two items pointing at aligned unsigned int's */
+/** Compare two items pointing at aligned unsigned int's.
+ *
+ *	This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
+ *	but #mdb_cmp_clong() is called instead if the data type is size_t.
+ */
 static int
 mdb_cmp_int(const MDB_val *a, const MDB_val *b)
 {
@@ -4799,13 +4829,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
 #endif
 }
 
-/** Compare two items pointing at size_t's of unknown alignment. */
-#ifdef MISALIGNED_OK
-# define mdb_cmp_clong mdb_cmp_long
-#else
-# define mdb_cmp_clong mdb_cmp_cint
-#endif
-
 /** Compare two items lexically */
 static int
 mdb_cmp_memn(const MDB_val *a, const MDB_val *b)
@@ -5741,15 +5764,21 @@ set1:
 					return rc;
 			}
 		} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
-			MDB_val d2;
+			MDB_val olddata;
-			if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS)
+			MDB_cmp_func *dcmp;
+			if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
 				return rc;
-			rc = mc->mc_dbx->md_dcmp(data, &d2);
+			dcmp = mc->mc_dbx->md_dcmp;
+#if UINT_MAX < SIZE_MAX
+			if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+				dcmp = mdb_cmp_clong;
+#endif
+			rc = dcmp(data, &olddata);
 			if (rc) {
 				if (op == MDB_GET_BOTH || rc > 0)
 					return MDB_NOTFOUND;
 				rc = 0;
-				*data = d2;
+				*data = olddata;
 			}
 
 		} else {
@@ -6259,16 +6288,17 @@ more:
 
 			/* Was a single item before, must convert now */
 			if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
+				MDB_cmp_func *dcmp;
 				/* Just overwrite the current item */
 				if (flags == MDB_CURRENT)
 					goto current;
-
+				dcmp = mc->mc_dbx->md_dcmp;
 #if UINT_MAX < SIZE_MAX
-				if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
+				if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
-					mc->mc_dbx->md_dcmp = mdb_cmp_clong;
+					dcmp = mdb_cmp_clong;
 #endif
 				/* does data match? */
-				if (!mc->mc_dbx->md_dcmp(data, &olddata)) {
+				if (!dcmp(data, &olddata)) {
 					if (flags & MDB_NODUPDATA)
 						return MDB_KEYEXIST;
 					/* overwrite it */
@@ -7088,6 +7118,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
 	mc->mc_snum = 0;
 	mc->mc_top = 0;
 	mc->mc_pg[0] = 0;
+	mc->mc_ki[0] = 0;
 	mc->mc_flags = 0;
 	if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
 		mdb_tassert(txn, mx != NULL);
@@ -7717,12 +7748,12 @@ mdb_rebalance(MDB_cursor *mc)
 						m3 = m2;
 					if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
 					if (m3->mc_pg[0] == mp) {
-						m3->mc_snum--;
-						m3->mc_top--;
 						for (i=0; i<m3->mc_snum; i++) {
 							m3->mc_pg[i] = m3->mc_pg[i+1];
 							m3->mc_ki[i] = m3->mc_ki[i+1];
 						}
+						m3->mc_snum--;
+						m3->mc_top--;
 					}
 				}
 			}
@@ -7790,9 +7821,23 @@ mdb_rebalance(MDB_cursor *mc)
 		if (mc->mc_ki[ptop] == 0) {
 			rc = mdb_page_merge(&mn, mc);
 		} else {
+			MDB_cursor dummy;
 			oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
 			mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
+			/* We want mdb_rebalance to find mn when doing fixups */
+			if (mc->mc_flags & C_SUB) {
+				dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
+				dummy.mc_xcursor = (MDB_xcursor *)&mn;
+			} else {
+				mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
+			}
 			rc = mdb_page_merge(mc, &mn);
+			if (mc->mc_flags & C_SUB)
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next;
+			else
+				mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
 			mdb_cursor_copy(&mn, mc);
 		}
 		mc->mc_flags &= ~C_EOF;
@@ -7819,6 +7864,13 @@ mdb_cursor_del0(MDB_cursor *mc)
 		MDB_cursor *m2, *m3;
 		MDB_dbi dbi = mc->mc_dbi;
 
+		/* DB is totally empty now, just bail out.
+		 * Other cursors adjustments were already done
+		 * by mdb_rebalance and aren't needed here.
+		 */
+		if (!mc->mc_snum)
+			return rc;
+
 		mp = mc->mc_pg[mc->mc_top];
 		nkeys = NUMKEYS(mp);
 

external/db_drivers/liblmdb64/mtest.c

@@ -45,19 +45,22 @@ int main(int argc,char * argv[])
 	    }
     
 		E(mdb_env_create(&env));
+		E(mdb_env_set_maxreaders(env, 1));
 		E(mdb_env_set_mapsize(env, 10485760));
 		E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));
+
 		E(mdb_txn_begin(env, NULL, 0, &txn));
-		E(mdb_open(txn, NULL, 0, &dbi));
+		E(mdb_dbi_open(txn, NULL, 0, &dbi));
    
 		key.mv_size = sizeof(int);
 		key.mv_data = sval;
-		data.mv_size = sizeof(sval);
-		data.mv_data = sval;
 
 		printf("Adding %d values\n", count);
 	    for (i=0;i<count;i++) {	
 			sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+			/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
+			data.mv_size = sizeof(sval);
+			data.mv_data = sval;
 			if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
 				j++;
 				data.mv_size = sizeof(sval);
@@ -68,7 +71,7 @@ int main(int argc,char * argv[])
 		E(mdb_txn_commit(txn));
 		E(mdb_env_stat(env, &mst));
 
-		E(mdb_txn_begin(env, NULL, 1, &txn));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 		E(mdb_cursor_open(txn, dbi, &cursor));
 		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 			printf("key: %p %.*s, data: %p %.*s\n",
@@ -97,7 +100,7 @@ int main(int argc,char * argv[])
 		printf("Deleted %d values\n", j);
 
 		E(mdb_env_stat(env, &mst));
-		E(mdb_txn_begin(env, NULL, 1, &txn));
+		E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 		E(mdb_cursor_open(txn, dbi, &cursor));
 		printf("Cursor next\n");
 		while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -128,6 +131,7 @@ int main(int argc,char * argv[])
 				(int) key.mv_size,  (char *) key.mv_data,
 				(int) data.mv_size, (char *) data.mv_data);
 
+		mdb_cursor_close(cursor);
 		mdb_txn_abort(txn);
 
 		printf("Deleting with cursor\n");
@@ -164,9 +168,9 @@ int main(int argc,char * argv[])
 				data.mv_data, (int) data.mv_size, (char *) data.mv_data);
 		}
 		mdb_cursor_close(cursor);
-		mdb_close(env, dbi);
-
 		mdb_txn_abort(txn);
+
+		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
 
 	return 0;

external/db_drivers/liblmdb64/mtest2.c

@@ -47,20 +47,22 @@ int main(int argc,char * argv[])
 	}
 
 	E(mdb_env_create(&env));
+	E(mdb_env_set_maxreaders(env, 1));
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
+	E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
    
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
-	data.mv_size = sizeof(sval);
-	data.mv_data = sval;
 
 	printf("Adding %d values\n", count);
 	for (i=0;i<count;i++) {	
 		sprintf(sval, "%03x %d foo bar", values[i], values[i]);
+		data.mv_size = sizeof(sval);
+		data.mv_data = sval;
 		if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
 			j++;
 	}
@@ -68,7 +70,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -97,7 +99,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -114,10 +116,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb64/mtest3.c

@@ -53,8 +53,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
+	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
 
 	key.mv_size = sizeof(int);
 	key.mv_data = kval;
@@ -73,7 +74,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -107,7 +108,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -124,10 +125,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb64/mtest4.c

@@ -51,8 +51,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
+	E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
 
 	key.mv_size = sizeof(int);
 	key.mv_data = kval;
@@ -72,7 +73,7 @@ int main(int argc,char * argv[])
 
 	/* there should be one full page of dups now.
 	 */
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -142,7 +143,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -159,10 +160,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb64/mtest5.c

@@ -53,8 +53,9 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
+	E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 
 	key.mv_size = sizeof(int);
@@ -75,7 +76,7 @@ int main(int argc,char * argv[])
 	E(mdb_txn_commit(txn));
 	E(mdb_env_stat(env, &mst));
 
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
 		printf("key: %p %.*s, data: %p %.*s\n",
@@ -109,7 +110,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -126,10 +127,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
-	mdb_env_close(env);
 
+	mdb_dbi_close(env, dbi);
+	mdb_env_close(env);
 	return 0;
 }

external/db_drivers/liblmdb64/mtest6.c

@@ -31,7 +31,7 @@ int main(int argc,char * argv[])
 	int i = 0, j = 0, rc;
 	MDB_env *env;
 	MDB_dbi dbi;
-	MDB_val key, data;
+	MDB_val key, data, sdata;
 	MDB_txn *txn;
 	MDB_stat mst;
 	MDB_cursor *cursor;
@@ -46,33 +46,37 @@ int main(int argc,char * argv[])
 	E(mdb_env_set_mapsize(env, 10485760));
 	E(mdb_env_set_maxdbs(env, 4));
 	E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
+
 	E(mdb_txn_begin(env, NULL, 0, &txn));
-	E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
+	E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	E(mdb_stat(txn, dbi, &mst));
 
 	sval = calloc(1, mst.ms_psize / 4);
 	key.mv_size = sizeof(long);
 	key.mv_data = &kval;
-	data.mv_size = mst.ms_psize / 4 - 30;
+	sdata.mv_size = mst.ms_psize / 4 - 30;
-	data.mv_data = sval;
+	sdata.mv_data = sval;
 
 	printf("Adding 12 values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+4;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	printf("Adding 12 more values, should yield 3 splits\n");
 	for (i=0;i<12;i++) {
 		kval = i*5+1;
 		sprintf(sval, "%08x", kval);
+		data = sdata;
 		(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
 	}
 	E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
@@ -110,7 +114,7 @@ int main(int argc,char * argv[])
 	printf("Deleted %d values\n", j);
 
 	E(mdb_env_stat(env, &mst));
-	E(mdb_txn_begin(env, NULL, 1, &txn));
+	E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
 	E(mdb_cursor_open(txn, dbi, &cursor));
 	printf("Cursor next\n");
 	while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@@ -127,9 +131,9 @@ int main(int argc,char * argv[])
 	}
 	CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
 	mdb_cursor_close(cursor);
-	mdb_close(env, dbi);
-
 	mdb_txn_abort(txn);
+
+	mdb_dbi_close(env, dbi);
 #endif
 	mdb_env_close(env);
 

external/db_drivers/liblmdb64/sample-mdb.txt

@@ -32,7 +32,7 @@ int main(int argc,char * argv[])
 	rc = mdb_env_create(&env);
 	rc = mdb_env_open(env, "./testdb", 0, 0664);
 	rc = mdb_txn_begin(env, NULL, 0, &txn);
-	rc = mdb_open(txn, NULL, 0, &dbi);
+	rc = mdb_dbi_open(txn, NULL, 0, &dbi);
 
 	key.mv_size = sizeof(int);
 	key.mv_data = sval;
@@ -56,7 +56,7 @@ int main(int argc,char * argv[])
 	mdb_cursor_close(cursor);
 	mdb_txn_abort(txn);
 leave:
-	mdb_close(env, dbi);
+	mdb_dbi_close(env, dbi);
 	mdb_env_close(env);
 	return 0;
 }

external/unbound/Makefile.in

@@ -168,7 +168,7 @@ HOST_OBJ=unbound-host.lo
 HOST_OBJ_LINK=$(HOST_OBJ) $(SLDNS_OBJ) $(COMPAT_OBJ_WITHOUT_CTIMEARC4) @WIN_HOST_OBJ_LINK@
 UBANCHOR_SRC=smallapp/unbound-anchor.c
 UBANCHOR_OBJ=unbound-anchor.lo
-UBANCHOR_OBJ_LINK=$(UBANCHOR_OBJ) \
+UBANCHOR_OBJ_LINK=$(UBANCHOR_OBJ) parseutil.lo \
 $(COMPAT_OBJ_WITHOUT_CTIME) @WIN_UBANCHOR_OBJ_LINK@
 TESTBOUND_SRC=testcode/testbound.c testcode/testpkts.c \
 daemon/worker.c daemon/acl_list.c daemon/daemon.c daemon/stats.c \
@@ -1175,7 +1175,7 @@ delayer.lo delayer.o: $(srcdir)/testcode/delayer.c config.h $(srcdir)/util/net_h
 unbound-control.lo unbound-control.o: $(srcdir)/smallapp/unbound-control.c config.h \
  $(srcdir)/util/log.h $(srcdir)/util/config_file.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h
 unbound-anchor.lo unbound-anchor.o: $(srcdir)/smallapp/unbound-anchor.c config.h $(srcdir)/libunbound/unbound.h \
- $(srcdir)/sldns/rrdef.h \
+ $(srcdir)/sldns/rrdef.h $(srcdir)/sldns/parseutil.h \
  
 petal.lo petal.o: $(srcdir)/testcode/petal.c config.h \
  

external/unbound/config.h.cmake.in

@@ -85,6 +85,10 @@
    `SSL_COMP_get_compression_methods', and to 0 if you don't. */
 #cmakedefine HAVE_DECL_SSL_COMP_GET_COMPRESSION_METHODS
 
+/* Define to 1 if you have the declaration of `SSL_CTX_set_ecdh_auto', and to
+   0 if you don't. */
+#cmakedefine HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
+
 /* Define to 1 if you have the declaration of `strlcat', and to 0 if you
    don't. */
 #cmakedefine HAVE_DECL_STRLCAT

external/unbound/config.h.in

@@ -82,6 +82,10 @@
    `SSL_COMP_get_compression_methods', and to 0 if you don't. */
 #undef HAVE_DECL_SSL_COMP_GET_COMPRESSION_METHODS
 
+/* Define to 1 if you have the declaration of `SSL_CTX_set_ecdh_auto', and to
+   0 if you don't. */
+#undef HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
+
 /* Define to 1 if you have the declaration of `strlcat', and to 0 if you
    don't. */
 #undef HAVE_DECL_STRLCAT

external/unbound/configure

@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.69 for unbound 1.5.4.
+# Generated by GNU Autoconf 2.69 for unbound 1.5.5.
 #
 # Report bugs to <unbound-bugs@nlnetlabs.nl>.
 #
@@ -590,8 +590,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='unbound'
 PACKAGE_TARNAME='unbound'
-PACKAGE_VERSION='1.5.4'
+PACKAGE_VERSION='1.5.5'
-PACKAGE_STRING='unbound 1.5.4'
+PACKAGE_STRING='unbound 1.5.5'
 PACKAGE_BUGREPORT='unbound-bugs@nlnetlabs.nl'
 PACKAGE_URL=''
 
@@ -1389,7 +1389,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures unbound 1.5.4 to adapt to many kinds of systems.
+\`configure' configures unbound 1.5.5 to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1454,7 +1454,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of unbound 1.5.4:";;
+     short | recursive ) echo "Configuration of unbound 1.5.5:";;
    esac
   cat <<\_ACEOF
 
@@ -1629,7 +1629,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-unbound configure 1.5.4
+unbound configure 1.5.5
 generated by GNU Autoconf 2.69
 
 Copyright (C) 2012 Free Software Foundation, Inc.
@@ -2338,7 +2338,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by unbound $as_me 1.5.4, which was
+It was created by unbound $as_me 1.5.5, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   $ $0 $@
@@ -2690,7 +2690,7 @@ UNBOUND_VERSION_MAJOR=1
 
 UNBOUND_VERSION_MINOR=5
 
-UNBOUND_VERSION_MICRO=4
+UNBOUND_VERSION_MICRO=5
 
 
 LIBUNBOUND_CURRENT=5
@@ -16684,7 +16684,7 @@ rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext conftest.$ac_ext
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for LibreSSL" >&5
 $as_echo_n "checking for LibreSSL... " >&6; }
-if grep OPENSSL_VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
+if grep VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
 	{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
 $as_echo "yes" >&6; }
 
@@ -16845,6 +16845,36 @@ fi
 cat >>confdefs.h <<_ACEOF
 #define HAVE_DECL_SK_SSL_COMP_POP_FREE $ac_have_decl
 _ACEOF
+ac_fn_c_check_decl "$LINENO" "SSL_CTX_set_ecdh_auto" "ac_cv_have_decl_SSL_CTX_set_ecdh_auto" "
+$ac_includes_default
+#ifdef HAVE_OPENSSL_ERR_H
+#include <openssl/err.h>
+#endif
+
+#ifdef HAVE_OPENSSL_RAND_H
+#include <openssl/rand.h>
+#endif
+
+#ifdef HAVE_OPENSSL_CONF_H
+#include <openssl/conf.h>
+#endif
+
+#ifdef HAVE_OPENSSL_ENGINE_H
+#include <openssl/engine.h>
+#endif
+#include <openssl/ssl.h>
+#include <openssl/evp.h>
+
+"
+if test "x$ac_cv_have_decl_SSL_CTX_set_ecdh_auto" = xyes; then :
+  ac_have_decl=1
+else
+  ac_have_decl=0
+fi
+
+cat >>confdefs.h <<_ACEOF
+#define HAVE_DECL_SSL_CTX_SET_ECDH_AUTO $ac_have_decl
+_ACEOF
 
 fi
 
@@ -18150,6 +18180,8 @@ esac
 fi
 
 
+LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
+
 ac_fn_c_check_func "$LINENO" "reallocarray" "ac_cv_func_reallocarray"
 if test "x$ac_cv_func_reallocarray" = xyes; then :
   $as_echo "#define HAVE_REALLOCARRAY 1" >>confdefs.h
@@ -18164,8 +18196,6 @@ esac
 fi
 
 
-LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
-
 if test "$USE_NSS" = "no"; then
 	ac_fn_c_check_func "$LINENO" "arc4random" "ac_cv_func_arc4random"
 if test "x$ac_cv_func_arc4random" = xyes; then :
@@ -18890,7 +18920,7 @@ _ACEOF
 
 
 
-version=1.5.4
+version=1.5.5
 
 date=`date +'%b %e, %Y'`
 
@@ -19405,7 +19435,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by unbound $as_me 1.5.4, which was
+This file was extended by unbound $as_me 1.5.5, which was
 generated by GNU Autoconf 2.69.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -19471,7 +19501,7 @@ _ACEOF
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
 ac_cs_version="\\
-unbound config.status 1.5.4
+unbound config.status 1.5.5
 configured by $0, generated by GNU Autoconf 2.69,
   with options \\"\$ac_cs_config\\"
 

external/unbound/configure.ac

@@ -10,7 +10,7 @@ sinclude(dnstap/dnstap.m4)
 # must be numbers. ac_defun because of later processing
 m4_define([VERSION_MAJOR],[1])
 m4_define([VERSION_MINOR],[5])
-m4_define([VERSION_MICRO],[4])
+m4_define([VERSION_MICRO],[5])
 AC_INIT(unbound, m4_defn([VERSION_MAJOR]).m4_defn([VERSION_MINOR]).m4_defn([VERSION_MICRO]), unbound-bugs@nlnetlabs.nl, unbound)
 AC_SUBST(UNBOUND_VERSION_MAJOR, [VERSION_MAJOR])
 AC_SUBST(UNBOUND_VERSION_MINOR, [VERSION_MINOR])
@@ -566,7 +566,7 @@ if test $USE_NSS = "no"; then
 ACX_WITH_SSL
 ACX_LIB_SSL
 AC_MSG_CHECKING([for LibreSSL])
-if grep OPENSSL_VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
+if grep VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
 	AC_MSG_RESULT([yes])
 	AC_DEFINE([HAVE_LIBRESSL], [1], [Define if we have LibreSSL])
 	# libressl provides these compat functions, but they may also be
@@ -578,7 +578,7 @@ fi
 AC_CHECK_HEADERS([openssl/conf.h],,, [AC_INCLUDES_DEFAULT])
 AC_CHECK_HEADERS([openssl/engine.h],,, [AC_INCLUDES_DEFAULT])
 AC_CHECK_FUNCS([OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode])
-AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free], [], [], [
+AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free,SSL_CTX_set_ecdh_auto], [], [], [
 AC_INCLUDES_DEFAULT
 #ifdef HAVE_OPENSSL_ERR_H
 #include <openssl/err.h>
@@ -998,9 +998,10 @@ AC_REPLACE_FUNCS(strlcat)
 AC_REPLACE_FUNCS(strlcpy)
 AC_REPLACE_FUNCS(memmove)
 AC_REPLACE_FUNCS(gmtime_r)
-AC_REPLACE_FUNCS(reallocarray)
+dnl without CTIME, ARC4-functions and without reallocarray.
 LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
 AC_SUBST(LIBOBJ_WITHOUT_CTIMEARC4)
+AC_REPLACE_FUNCS(reallocarray)
 if test "$USE_NSS" = "no"; then
 	AC_REPLACE_FUNCS(arc4random)
 	AC_REPLACE_FUNCS(arc4random_uniform)

external/unbound/daemon/daemon.c

@@ -399,6 +399,12 @@ daemon_create_workers(struct daemon* daemon)
 	verbose(VERB_ALGO, "total of %d outgoing ports available", numport);
 	
 	daemon->num = (daemon->cfg->num_threads?daemon->cfg->num_threads:1);
+	if(daemon->reuseport && (int)daemon->num < (int)daemon->num_ports) {
+		log_warn("cannot reduce num-threads to %d because so-reuseport "
+			"so continuing with %d threads.", (int)daemon->num,
+			(int)daemon->num_ports);
+		daemon->num = (int)daemon->num_ports;
+	}
 	daemon->workers = (struct worker**)calloc((size_t)daemon->num, 
 		sizeof(struct worker*));
 	if(daemon->cfg->dnstap) {
@@ -464,7 +470,7 @@ thread_start(void* arg)
 #endif
 #ifdef SO_REUSEPORT
 	if(worker->daemon->cfg->so_reuseport)
-		port_num = worker->thread_num;
+		port_num = worker->thread_num % worker->daemon->num_ports;
 	else
 		port_num = 0;
 #endif

external/unbound/daemon/remote.c

@@ -243,9 +243,9 @@ daemon_remote_create(struct config_file* cfg)
 		goto setup_error;
 	}
 	verbose(VERB_ALGO, "setup SSL certificates");
-	if (!SSL_CTX_use_certificate_file(rc->ctx,s_cert,SSL_FILETYPE_PEM)) {
+	if (!SSL_CTX_use_certificate_chain_file(rc->ctx,s_cert)) {
 		log_err("Error for server-cert-file: %s", s_cert);
-		log_crypto_err("Error in SSL_CTX use_certificate_file");
+		log_crypto_err("Error in SSL_CTX use_certificate_chain_file");
 		goto setup_error;
 	}
 	if(!SSL_CTX_use_PrivateKey_file(rc->ctx,s_key,SSL_FILETYPE_PEM)) {
@@ -258,6 +258,23 @@ daemon_remote_create(struct config_file* cfg)
 		log_crypto_err("Error in SSL_CTX check_private_key");
 		goto setup_error;
 	}
+#if HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
+	if(!SSL_CTX_set_ecdh_auto(rc->ctx,1)) {
+		log_crypto_err("Error in SSL_CTX_ecdh_auto, not enabling ECDHE");
+	}
+#elif defined(USE_ECDSA)
+	if(1) {
+		EC_KEY *ecdh = EC_KEY_new_by_curve_name (NID_X9_62_prime256v1);
+		if (!ecdh) {
+			log_crypto_err("could not find p256, not enabling ECDHE");
+		} else {
+			if (1 != SSL_CTX_set_tmp_ecdh (rc->ctx, ecdh)) {
+				log_crypto_err("Error in SSL_CTX_set_tmp_ecdh, not enabling ECDHE");
+			}
+			EC_KEY_free (ecdh);
+		}
+	}
+#endif
 	if(!SSL_CTX_load_verify_locations(rc->ctx, s_cert, NULL)) {
 		log_crypto_err("Error setting up SSL_CTX verify locations");
 	setup_error:
@@ -1683,6 +1700,7 @@ parse_delegpt(SSL* ssl, char* args, uint8_t* nm, int allow_names)
 			}
 		}
 	}
+	dp->has_parent_side_NS = 1;
 	return dp;
 }
 

external/unbound/daemon/worker.c

@@ -568,7 +568,7 @@ answer_from_cache(struct worker* worker, struct query_info* qinfo,
 	if(rep->an_numrrsets > 0 && (rep->rrsets[0]->rk.type == 
 		htons(LDNS_RR_TYPE_CNAME) || rep->rrsets[0]->rk.type == 
 		htons(LDNS_RR_TYPE_DNAME))) {
-		if(!reply_check_cname_chain(rep)) {
+		if(!reply_check_cname_chain(qinfo, rep)) {
 			/* cname chain invalid, redo iterator steps */
 			verbose(VERB_ALGO, "Cache reply: cname chain broken");
 		bail_out:

external/unbound/doc/Changelog

@@ -1,3 +1,73 @@
+13 August 2015: Wouter
+	- 5011 implementation does not insist on all algorithms, when
+	  harden-algo-downgrade is turned off.
+	- Reap the child process that libunbound spawns.
+
+11 August 2015: Wouter
+	- Fix #694: configure script does not detect LibreSSL 2.2.2
+
+4 August 2015: Wouter
+	- Document that local-zone nodefault matches exactly and transparent
+	  can be used to release a subzone.
+
+3 August 2015: Wouter
+	- Document in the manual more text about configuring locally served
+	  zones.
+	- Fix 5011 anchor update timer after reload.
+	- Fix mktime in unbound-anchor not using UTC.
+
+30 July 2015: Wouter
+	- please afl-gcc (llvm) for uninitialised variable warning.
+	- Added permit-small-holddown config to debug fast 5011 rollover.
+
+24 July 2015: Wouter
+	- Fix #690: Reload fails when so-reuseport is yes after changing
+	  num-threads.
+	- iana portlist update.
+
+21 July 2015: Wouter
+	- Fix configure to detect SSL_CTX_set_ecdh_auto.
+	- iana portlist update.
+
+20 July 2015: Wouter
+	- Enable ECDHE for servers.  Where available, use
+	  SSL_CTX_set_ecdh_auto() for TLS-wrapped server configurations to
+	  enable ECDHE.  Otherwise, manually offer curve p256.
+	  Client connections should automatically use ECDHE when available.
+	  (thanks Daniel Kahn Gillmor)
+
+18 July 2015: Willem
+	- Allow certificate chain files to allow for intermediate certificates.
+	  (thanks Daniel Kahn Gillmor)
+
+13 July 2015: Wouter
+	- makedist produces sha1 and sha256 files for created binaries too.
+
+9 July 2015: Wouter
+	- 1.5.4 release tag
+	- trunk has 1.5.5 in development.
+	- Fix #681: Setting forwarders with unbound-control forward
+	  implicitly turns on forward-first.
+
+29 June 2015: Wouter
+	- iana portlist update.
+	- Fix alloc with log for allocation size checks.
+
+26 June 2015: Wouter
+	- Fix #677 Fix DNAME responses from cache that failed internal chain
+	  test.
+	- iana portlist update.
+
+22 June 2015: Wouter
+	- Fix #677 Fix CNAME corresponding to a DNAME was checked incorrectly
+	  and was therefore always synthesized (thanks to Valentin Dietrich).
+
+4 June 2015: Wouter
+	- RFC 7553 RR type URI support, is now enabled by default.
+
+2 June 2015: Wouter
+	- Fix #674: Do not free pointers given by getenv.
+
 29 May 2015: Wouter
 	- Fix that unparseable error responses are ratelimited.
 	- SOA negative TTL is capped at minimumttl in its rdata section.

external/unbound/doc/example.conf.in

@@ -444,6 +444,9 @@ server:
 	# If the value 0 is given, missing anchors are not removed.
 	# keep-missing: 31622400 # 366 days
 
+	# debug option that allows very small holddown times for key rollover
+	# permit-small-holddown: no
+
 	# the amount of memory to use for the key cache.
 	# plain value in bytes or you can append k, m or G. default is "4Mb". 
 	# key-cache-size: 4m
@@ -623,6 +626,8 @@ remote-control:
 # nameservers by hostname or by ipaddress. If you set stub-prime to yes, 
 # the list is treated as priming hints (default is no).
 # With stub-first yes, it attempts without the stub if it fails.
+# Consider adding domain-insecure: name and local-zone: name nodefault
+# to the server: section if the stub is a locally served zone.
 # stub-zone:
 #	name: "example.com"
 #	stub-addr: 192.0.2.68

external/unbound/doc/unbound.conf.5.in

@@ -801,6 +801,10 @@ mechanism work with zones that perform regular (non\-5011) rollovers.
 The default is 366 days.  The value 0 does not remove missing anchors,
 as per the RFC.
 .TP
+.B permit\-small\-holddown: \fI<yes or no>
+Debug option that allows the autotrust 5011 rollover timers to assume
+very small values.  Default is no.
+.TP
 .B key\-cache\-size: \fI<number>
 Number of bytes size of the key cache. Default is 4 megabytes.
 A plain number is in bytes, append 'k', 'm' or 'g' for kilobytes, megabytes
@@ -895,7 +899,8 @@ infected machines without answering the queries.
 Used to turn off default contents for AS112 zones. The other types
 also turn off default contents for the zone. The 'nodefault' option 
 has no other effect than turning off default contents for the 
-given zone.
+given zone.  Use \fInodefault\fR if you use exactly that zone, if you want to
+use a subzone, use \fItransparent\fR.
 .P
 The default zones are localhost, reverse 127.0.0.1 and ::1, and the AS112
 zones. The AS112 zones are reverse DNS zones for private use and reserved
@@ -1124,6 +1129,12 @@ bit on replies for the private zone (authoritative servers do not set the
 AD bit).  This setup makes unbound capable of answering queries for the 
 private zone, and can even set the AD bit ('authentic'), but the AA 
 ('authoritative') bit is not set on these replies. 
+.P
+Consider adding \fBserver:\fR statements for \fBdomain\-insecure:\fR and
+for \fBlocal\-zone:\fI name nodefault\fR for the zone if it is a locally
+served zone.  The insecure clause stops DNSSEC from invalidating the
+zone.  The local zone nodefault (or \fItransparent\fR) clause makes the
+(reverse\-) zone bypass unbound's filtering of RFC1918 zones.
 .TP
 .B name: \fI<domain name>
 Name of the stub zone.

external/unbound/iterator/iter_scrub.c

@@ -372,7 +372,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
 				/* check next cname */
 				uint8_t* t = NULL;
 				size_t tlen = 0;
-				if(!parse_get_cname_target(rrset, &t, &tlen))
+				if(!parse_get_cname_target(nx, &t, &tlen))
 					return 0;
 				if(dname_pkt_compare(pkt, alias, t) == 0) {
 					/* it's OK and better capitalized */

external/unbound/libunbound/libunbound.c

@@ -65,6 +65,9 @@
 #ifdef HAVE_PTHREAD
 #include <signal.h>
 #endif
+#ifdef HAVE_SYS_WAIT_H
+#include <sys/wait.h>
+#endif
 
 #if defined(UB_ON_WINDOWS) && defined (HAVE_WINDOWS_H)
 #include <windows.h>
@@ -218,6 +221,12 @@ static void ub_stop_bg(struct ub_ctx* ctx)
 			ub_thread_join(ctx->bg_tid);
 		} else {
 			lock_basic_unlock(&ctx->cfglock);
+#ifndef UB_ON_WINDOWS
+			if(waitpid(ctx->bg_pid, NULL, 0) == -1) {
+				if(verbosity > 2)
+					log_err("waitpid: %s", strerror(errno));
+			}
+#endif
 		}
 	}
 	else {
@@ -1028,6 +1037,7 @@ ub_ctx_hosts(struct ub_ctx* ctx, const char* fname)
 					"\\hosts");
 				retval=ub_ctx_hosts(ctx, buf);
 			}
+			free(name);
 			return retval;
 		}
 		return UB_READFILE;

external/unbound/makedist.sh

@@ -136,6 +136,43 @@ create_temp_dir () {
     cd $temp_dir
 }
 
+# pass filename as $1 arg.
+# creates file.sha1 and file.sha256
+storehash () {
+    case $OSTYPE in
+        linux*)
+                sha=`sha1sum $1 |  awk '{ print $1 }'`
+                sha256=`sha256sum $1 |  awk '{ print $1 }'`
+                ;;
+        freebsd*)
+                sha=`sha1 $1 |  awk '{ print $5 }'`
+                sha256=`sha256 $1 |  awk '{ print $5 }'`
+                ;;
+	*)
+		# in case $OSTYPE is gone.
+		case `uname` in
+		Linux*)
+		  sha=`sha1sum $1 |  awk '{ print $1 }'`
+		  sha256=`sha256sum $1 |  awk '{ print $1 }'`
+		  ;;
+		FreeBSD*)
+		  sha=`sha1 $1 |  awk '{ print $5 }'`
+		  sha256=`sha256 $1 |  awk '{ print $5 }'`
+		  ;;
+		*)
+		  sha=`sha1sum $1 |  awk '{ print $1 }'`
+		  sha256=`sha256sum $1 |  awk '{ print $1 }'`
+		  ;;
+		esac
+                ;;
+    esac
+    echo $sha > $1.sha1
+    echo $sha256 > $1.sha256
+    echo "hash of $1.{sha1,sha256}"
+    echo "sha1 $sha"
+    echo "sha256 $sha256"
+}
+
 
 SNAPSHOT="no"
 RC="no"
@@ -311,6 +348,8 @@ if [ "$DOWIN" = "yes" ]; then
 	    mv unbound-$version.zip $cwd/.
 	    cleanup
     fi
+    storehash unbound_setup_$version.exe
+    storehash unbound-$version.zip
     ls -lG unbound_setup_$version.exe
     ls -lG unbound-$version.zip
     info "Done"
@@ -411,36 +450,7 @@ tar czf ../unbound-$version.tar.gz unbound-$version || error_cleanup "Failed to
 
 cleanup
 
-case $OSTYPE in
+storehash unbound-$version.tar.gz
-        linux*)
-                sha=`sha1sum unbound-$version.tar.gz |  awk '{ print $1 }'`
-                sha256=`sha256sum unbound-$version.tar.gz |  awk '{ print $1 }'`
-                ;;
-        freebsd*)
-                sha=`sha1  unbound-$version.tar.gz |  awk '{ print $5 }'`
-                sha256=`sha256  unbound-$version.tar.gz |  awk '{ print $5 }'`
-                ;;
-	*)
-		# in case $OSTYPE is gone.
-		case `uname` in
-		Linux*)
-		  sha=`sha1sum unbound-$version.tar.gz |  awk '{ print $1 }'`
-		  sha256=`sha256sum unbound-$version.tar.gz |  awk '{ print $1 }'`
-		  ;;
-		FreeBSD*)
-		  sha=`sha1  unbound-$version.tar.gz |  awk '{ print $5 }'`
-		  sha256=`sha256  unbound-$version.tar.gz |  awk '{ print $5 }'`
-		  ;;
-		*)
-		  sha=`sha1sum unbound-$version.tar.gz |  awk '{ print $1 }'`
-		  sha256=`sha256sum unbound-$version.tar.gz |  awk '{ print $1 }'`
-		  ;;
-		esac
-                ;;
-esac
-echo $sha > unbound-$version.tar.gz.sha1
-echo $sha256 > unbound-$version.tar.gz.sha256
 
 info "Unbound distribution created successfully."
-info "SHA1sum: $sha"
 

external/unbound/services/cache/dns.c

@@ -505,7 +505,7 @@ tomsg(struct module_env* env, struct query_info* q, struct reply_info* r,
 		return NULL;
 	if(r->an_numrrsets > 0 && (r->rrsets[0]->rk.type == htons(
 		LDNS_RR_TYPE_CNAME) || r->rrsets[0]->rk.type == htons(
-		LDNS_RR_TYPE_DNAME)) && !reply_check_cname_chain(r)) {
+		LDNS_RR_TYPE_DNAME)) && !reply_check_cname_chain(q, r)) {
 		/* cname chain is now invalid, reconstruct msg */
 		rrset_array_unlock(r->ref, r->rrset_count);
 		return NULL;

external/unbound/sldns/rrdef.c

@@ -213,13 +213,11 @@ static const sldns_rdf_type type_eui48_wireformat[] = {
 static const sldns_rdf_type type_eui64_wireformat[] = {
 	LDNS_RDF_TYPE_EUI64
 };
-#ifdef DRAFT_RRTYPES
 static const sldns_rdf_type type_uri_wireformat[] = {
 	LDNS_RDF_TYPE_INT16,
 	LDNS_RDF_TYPE_INT16,
 	LDNS_RDF_TYPE_LONG_STR
 };
-#endif
 static const sldns_rdf_type type_caa_wireformat[] = {
 	LDNS_RDF_TYPE_INT8,
 	LDNS_RDF_TYPE_TAG,
@@ -590,12 +588,8 @@ static sldns_rr_descriptor rdata_field_descriptors[] = {
 	/* ANY: A request for all (available) records */
 {LDNS_RR_TYPE_ANY, "ANY", 1, 1, type_0_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
 
-#ifdef DRAFT_RRTYPES
 	/* 256 */
 	{LDNS_RR_TYPE_URI, "URI", 3, 3, type_uri_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
-#else
-{LDNS_RR_TYPE_NULL, "TYPE256", 1, 1, type_0_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
-#endif
 	/* 257 */
 	{LDNS_RR_TYPE_CAA, "CAA", 3, 3, type_caa_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
 

external/unbound/sldns/rrdef.h

@@ -220,8 +220,7 @@ enum sldns_enum_rr_type
 	LDNS_RR_TYPE_MAILA = 254,
 	/**  any type (wildcard) */
 	LDNS_RR_TYPE_ANY = 255,
-	/** draft-faltstrom-uri-06 */
+	LDNS_RR_TYPE_URI = 256, /* RFC 7553 */
-	LDNS_RR_TYPE_URI = 256,
 	LDNS_RR_TYPE_CAA = 257, /* RFC 6844 */
 
 	/** DNSSEC Trust Authorities */

external/unbound/smallapp/unbound-anchor.c

@@ -117,6 +117,7 @@
 #include "config.h"
 #include "libunbound/unbound.h"
 #include "sldns/rrdef.h"
+#include "sldns/parseutil.h"
 #include <expat.h>
 #ifndef HAVE_EXPAT_H
 #error "need libexpat to parse root-anchors.xml file."
@@ -1328,7 +1329,7 @@ xml_convertdate(const char* str)
 		/* but ignore, (lenient) */
 	}
 
-	t = mktime(&tm);
+	t = sldns_mktime_from_utc(&tm);
 	if(t == (time_t)-1) {
 		if(verb) printf("xml_convertdate mktime failure\n");
 		return 0;

external/unbound/smallapp/unbound-control.c

@@ -161,7 +161,7 @@ setup_ctx(struct config_file* cfg)
         if(cfg->remote_control_use_cert) {
 		if(!(SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3) & SSL_OP_NO_SSLv3))
 			ssl_err("could not set SSL_OP_NO_SSLv3");
-		if(!SSL_CTX_use_certificate_file(ctx,c_cert,SSL_FILETYPE_PEM) ||
+		if(!SSL_CTX_use_certificate_chain_file(ctx,c_cert) ||
 		    !SSL_CTX_use_PrivateKey_file(ctx,c_key,SSL_FILETYPE_PEM)
 		    || !SSL_CTX_check_private_key(ctx))
 			ssl_err("Error setting up SSL_CTX client key and cert");

external/unbound/testcode/petal.c

@@ -236,12 +236,28 @@ setup_ctx(char* key, char* cert)
 	if(!ctx) print_exit("out of memory");
 	(void)SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2);
 	(void)SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3);
-	if(!SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM))
+	if(!SSL_CTX_use_certificate_chain_file(ctx, cert))
 		print_exit("cannot read cert");
 	if(!SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM))
 		print_exit("cannot read key");
 	if(!SSL_CTX_check_private_key(ctx))
 		print_exit("private key is not correct");
+#if HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
+	if (!SSL_CTX_set_ecdh_auto(ctx,1))
+		if(verb>=1) printf("failed to set_ecdh_auto, not enabling ECDHE\n");
+#elif defined(USE_ECDSA)
+	if(1) {
+		EC_KEY *ecdh = EC_KEY_new_by_curve_name (NID_X9_62_prime256v1);
+		if (!ecdh) {
+			if(verb>=1) printf("could not find p256, not enabling ECDHE\n");
+		} else {
+			if (1 != SSL_CTX_set_tmp_ecdh (ctx, ecdh)) {
+				if(verb>=1) printf("Error in SSL_CTX_set_tmp_ecdh, not enabling ECDHE\n");
+			}
+			EC_KEY_free(ecdh);
+		}
+	}
+#endif
 	if(!SSL_CTX_load_verify_locations(ctx, cert, NULL))
 		print_exit("cannot load cert verify locations");
 	return ctx;

external/unbound/testdata/fwd_any.rpl

@@ -1,161 +0,0 @@
-; This is a comment.
-; config options go here.
-forward-zone: name: "." forward-addr: 216.0.0.1
-CONFIG_END
-
-SCENARIO_BEGIN Test query and cache with type ANY
-RANGE_BEGIN 0 1000
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A 10.20.30.40
-SECTION AUTHORITY
-www.example.com. IN NS ns.example.com.
-SECTION ADDITIONAL
-ns.example.com. IN A 10.20.30.50
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN ANY
-SECTION ANSWER
-;; different type in this answer.
-www.example.com. IN TXT "text"
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN AAAA
-SECTION ANSWER
-www.example.com. IN AAAA ::5
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qname qtype
-ADJUST copy_id
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.foo.com. IN ANY
-SECTION ANSWER
-www.foo.com. IN A 1.2.3.77
-www.foo.com. IN AAAA ::77
-ENTRY_END
-
-RANGE_END
-
-STEP 10 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; unneccesary nothing steps.
-STEP 20 NOTHING
-STEP 30 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH opcode qname qtype
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A 10.20.30.40
-ENTRY_END
-
-; test cache synthesis
-STEP 40 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN ANY
-ENTRY_END
-STEP 50 NOTHING
-STEP 60 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH opcode qname qtype
-SECTION QUESTION
-www.example.com. IN ANY
-SECTION ANSWER
-www.example.com. IN A 10.20.30.40
-ENTRY_END
-
-; and again
-; the synthesized result itself is not added to the cache
-STEP 62 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN ANY
-ENTRY_END
-STEP 63 NOTHING
-STEP 64 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH opcode qname qtype
-SECTION QUESTION
-www.example.com. IN ANY
-SECTION ANSWER
-www.example.com. IN A 10.20.30.40
-ENTRY_END
-
-; AAAA lookup to add more data in cache
-STEP 70 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN AAAA
-ENTRY_END
-STEP 80 NOTHING
-STEP 90 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH opcode qname qtype
-SECTION QUESTION
-www.example.com. IN AAAA
-SECTION ANSWER
-www.example.com. IN AAAA ::5
-ENTRY_END
-
-; test cache synthesis of AAAA, and two rrsets.
-STEP 100 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN ANY
-ENTRY_END
-STEP 110 NOTHING
-STEP 120 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH opcode qname qtype
-SECTION QUESTION
-www.example.com. IN ANY
-SECTION ANSWER
-www.example.com. IN A 10.20.30.40
-www.example.com. IN AAAA ::5
-ENTRY_END
-
-; test query that is not synthesized from cache.
-STEP 130 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.foo.com. IN ANY
-ENTRY_END
-STEP 140 NOTHING
-STEP 150 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH opcode qname qtype
-SECTION QUESTION
-www.foo.com. IN ANY
-SECTION ANSWER
-www.foo.com. IN A 1.2.3.77
-www.foo.com. IN AAAA ::77
-ENTRY_END
-
-SCENARIO_END

external/unbound/testdata/iter_domain_sale.rpl

@@ -1,273 +0,0 @@
-; config options
-server:
-	target-fetch-policy: "0 0 0 0 0"
-
-stub-zone:
-	name: "."
-	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
-CONFIG_END
-
-SCENARIO_BEGIN Test resolver with a domain sale
-; and the old operator is nasty, keeps running his server with the old data.
-; and lots of lookups keep going towards the domain.
-; eventually, the NS record has to timeout.
-
-; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
-	ADDRESS 193.0.14.129 
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-. IN NS
-SECTION ANSWER
-. IN NS	K.ROOT-SERVERS.NET.
-SECTION ADDITIONAL
-K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode subdomain
-ADJUST copy_id copy_query
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN A
-SECTION AUTHORITY
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net. (before sale of domain)
-RANGE_BEGIN 0 20
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode subdomain
-ADJUST copy_id copy_query
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net. (after sale of domain)
-RANGE_BEGIN 30 200
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode subdomain
-ADJUST copy_id copy_query
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	8.8.8.8
-ENTRY_END
-RANGE_END
-
-; ns.example.com. first owner
-RANGE_BEGIN 0 200
-	ADDRESS 1.2.3.4
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qname
-ADJUST copy_id copy_query
-REPLY QR AA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-; nxdomains for any name,type
-; last in RANGE so that it matches everything left over.
-; it includes the NS record.
-ENTRY_BEGIN
-MATCH opcode
-ADJUST copy_id copy_query
-REPLY QR AA NXDOMAIN
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-SECTION AUTHORITY
-example.com.	3600 IN SOA	a. b. 1 2 3 4 5
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; ns.example.com. new owner
-RANGE_BEGIN 0 200
-	ADDRESS 8.8.8.8
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	8.8.8.8
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	88.88.88.88
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	8.8.8.8
-ENTRY_END
-RANGE_END
-
-; Fetch the old record from the old owner.
-STEP 1 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 5 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-; the domain is sold (right at this time).
-; but the information stays in the cache.
-
-; after 1800 secs still the cached answer
-STEP 20 TIME_PASSES ELAPSE 1800
-
-STEP 30 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 40 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 1800 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	1800 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	1800 	IN 	A	1.2.3.4
-ENTRY_END
-
-; and ask another query
-STEP 50 QUERY 
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-nx1.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 60 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NXDOMAIN
-SECTION QUESTION
-nx1.example.com. IN A
-SECTION ANSWER
-SECTION AUTHORITY
-; at TTL 5 because TTL is capped at min-ttl of 5 in rdata of SOA
-example.com.	5 IN SOA	a. b. 1 2 3 4 5
-example.com.	1800 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	1800 	IN 	A	1.2.3.4
-ENTRY_END
-
-; after another 1900 seconds the domain must have timed out.
-STEP 70 TIME_PASSES ELAPSE 1900
-
-; the NS record should have timed out.
-STEP 80 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 90 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	88.88.88.88
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	8.8.8.8
-ENTRY_END
-
-SCENARIO_END

external/unbound/testdata/iter_domain_sale_nschange.rpl

@@ -1,342 +0,0 @@
-; config options
-server:
-	target-fetch-policy: "0 0 0 0 0"
-
-stub-zone:
-	name: "."
-	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
-CONFIG_END
-
-SCENARIO_BEGIN Test resolver with a domain sale and NS changes
-; and the old operator is nasty, keeps running his server with the old data.
-; and lots of lookups keep going towards the domain.
-; and the old server is changing the NS record of the old domain.
-
-; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
-	ADDRESS 193.0.14.129 
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-. IN NS
-SECTION ANSWER
-. IN NS	K.ROOT-SERVERS.NET.
-SECTION ADDITIONAL
-K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode subdomain
-ADJUST copy_id copy_query
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN A
-SECTION AUTHORITY
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net. (before sale of domain)
-RANGE_BEGIN 0 20
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode subdomain
-ADJUST copy_id copy_query
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net. (after sale of domain)
-RANGE_BEGIN 30 200
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode subdomain
-ADJUST copy_id copy_query
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	8.8.8.8
-ENTRY_END
-RANGE_END
-
-; ns.example.com. first owner
-RANGE_BEGIN 0 30
-	ADDRESS 1.2.3.4
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qname
-ADJUST copy_id copy_query
-REPLY QR AA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-; nxdomains for any name,type
-; last in RANGE so that it matches everything left over.
-; it includes the NS record.
-ENTRY_BEGIN
-MATCH opcode
-ADJUST copy_id copy_query
-REPLY QR AA NXDOMAIN
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-SECTION AUTHORITY
-example.com.	3600 IN SOA	a. b. 1 2 3 4 5
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; ns.example.com. first owner, NS changed
-RANGE_BEGIN 40 200
-	ADDRESS 1.2.3.4
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.	IN NS	nsb.example.com.
-SECTION ADDITIONAL
-nsb.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qname
-ADJUST copy_id copy_query
-REPLY QR AA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	3600 IN NS	nsb.example.com.
-SECTION ADDITIONAL
-nsb.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-; nxdomains for any name,type
-; last in RANGE so that it matches everything left over.
-; it includes the NS record.
-ENTRY_BEGIN
-MATCH opcode
-ADJUST copy_id copy_query
-REPLY QR AA NXDOMAIN
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-SECTION AUTHORITY
-example.com.	3600 IN SOA	a. b. 1 2 3 4 5
-example.com.	3600 IN NS	nsb.example.com.
-SECTION ADDITIONAL
-nsb.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; ns.example.com. new owner
-RANGE_BEGIN 0 200
-	ADDRESS 8.8.8.8
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	8.8.8.8
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	88.88.88.88
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	8.8.8.8
-ENTRY_END
-RANGE_END
-
-; Fetch the old record from the old owner.
-STEP 1 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 5 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-; the domain is sold (right at this time).
-; but the information stays in the cache.
-
-; after 1800 secs still the cached answer
-STEP 20 TIME_PASSES ELAPSE 1800
-
-STEP 30 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 40 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 1800 IN A	10.20.30.40
-SECTION AUTHORITY
-example.com.	1800 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	1800 	IN 	A	1.2.3.4
-ENTRY_END
-
-; and ask another query
-STEP 50 QUERY 
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-nx1.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 60 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NXDOMAIN
-SECTION QUESTION
-nx1.example.com. IN A
-SECTION ANSWER
-SECTION AUTHORITY
-; at TTL 5 because TTL capped at ttl of minttl in rdata of SOA.
-example.com.	5 IN SOA	a. b. 1 2 3 4 5
-example.com.	3600 IN NS	nsb.example.com.
-SECTION ADDITIONAL
-nsb.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-STEP 62 QUERY 
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-nx1.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 63 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NXDOMAIN
-SECTION QUESTION
-nx1.example.com. IN A
-SECTION ANSWER
-SECTION AUTHORITY
-; at TTL 5 because TTL capped at ttl of minttl in rdata of SOA.
-example.com.	5 IN SOA	a. b. 1 2 3 4 5
-example.com.	1800 IN NS	nsb.example.com.
-SECTION ADDITIONAL
-nsb.example.com.	3600 	IN 	A	1.2.3.4
-ENTRY_END
-
-; after another 1900 seconds the domain must have timed out.
-STEP 70 TIME_PASSES ELAPSE 1900
-
-; the NS record should have timed out.
-STEP 80 QUERY
-ENTRY_BEGIN
-REPLY RD
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-; recursion happens here.
-STEP 90 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all ttl
-REPLY QR RD RA NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. 3600 IN A	88.88.88.88
-SECTION AUTHORITY
-example.com.	3600 IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.	3600 	IN 	A	8.8.8.8
-ENTRY_END
-
-SCENARIO_END

external/unbound/testdata/val_spurious_ns.rpl

@@ -1,151 +0,0 @@
-; config options
-; The island of trust is at example.com
-server:
-	trust-anchor: "example.com.    3600    IN      DS      2854 3 1 46e4ffc6e9a4793b488954bd3f0cc6af0dfb201b"
-	val-override-date: "20070916134226"
-	target-fetch-policy: "0 0 0 0 0"
-
-stub-zone:
-	name: "."
-	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
-CONFIG_END
-
-SCENARIO_BEGIN Test validator with spurious unsigned NS in auth section 
-
-; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
-	ADDRESS 193.0.14.129 
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-. IN NS
-SECTION ANSWER
-. IN NS	K.ROOT-SERVERS.NET.
-SECTION ADDITIONAL
-K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION AUTHORITY
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net.
-RANGE_BEGIN 0 100
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.    IN NS   a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.     IN      A       192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; ns.example.com.
-RANGE_BEGIN 0 100
-	ADDRESS 1.2.3.4
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.    IN NS   ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-SECTION ADDITIONAL
-ns.example.com.         IN      A       1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ENTRY_END
-
-; response to DNSKEY priming query
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN DNSKEY
-SECTION ANSWER
-example.com.    3600    IN      DNSKEY  256 3 3 ALXLUsWqUrY3JYER3T4TBJII s70j+sDS/UT2QRp61SE7S3E EXopNXoFE73JLRmvpi/UrOO/Vz4Se 6wXv/CYCKjGw06U4WRgR YXcpEhJROyNapmdIKSx hOzfLVE1gqA0PweZR8d tY3aNQSRn3sPpwJr6Mi /PqQKAMMrZ9ckJpf1+b QMOOvxgzz2U1GS18b3y ZKcgTMEaJzd/GZYzi/B N2DzQ0MsrSwYXfsNLFO Bbs8PJMW4LYIxeeOe6rUgkWOF 7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}
-example.com.    3600    IN      RRSIG   DNSKEY 3 2 3600 20070926134802 20070829134802 2854 example.com. MCwCFG1yhRNtTEa3Eno2zhVVuy2EJX3wAhQeLyUp6+UXcpC5qGNu9tkrTEgPUg== ;{id = 2854}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ENTRY_END
-
-; response to query of interest
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A	10.20.30.40
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-;example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-www.example.com.        3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
-ENTRY_END
-RANGE_END
-
-STEP 1 QUERY
-ENTRY_BEGIN
-REPLY RD DO
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-
-; recursion happens here.
-STEP 10 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all
-REPLY QR RD RA AD DO NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A	10.20.30.40
-www.example.com.        3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
-SECTION AUTHORITY
-; removed by spurious NS record removal code
-;;example.com.	IN NS	ns.example.com.
-;;example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
-ENTRY_END
-
-SCENARIO_END

external/unbound/testdata/val_ta_algo_dnskey_dp.rpl

@@ -1,182 +0,0 @@
-; config options
-; The island of trust is at example.com
-server:
-	trust-anchor: "example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}"
-	trust-anchor: "example.com.	3600	IN	DS	30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512"
-	val-override-date: "20070916134226"
-	target-fetch-policy: "0 0 0 0 0"
-	harden-algo-downgrade: no
-
-stub-zone:
-	name: "."
-	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
-CONFIG_END
-
-SCENARIO_BEGIN Test validator with multiple algorithm trust anchor without harden 
-
-; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
-	ADDRESS 193.0.14.129 
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-. IN NS
-SECTION ANSWER
-. IN NS	K.ROOT-SERVERS.NET.
-SECTION ADDITIONAL
-K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION AUTHORITY
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net.
-RANGE_BEGIN 0 100
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.    IN NS   a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.     IN      A       192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; ns.example.com.
-RANGE_BEGIN 0 100
-	ADDRESS 1.2.3.4
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.    IN NS   ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.         IN      A       1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-ns.example.com. IN AAAA
-SECTION ANSWER
-SECTION AUTHORITY
-example.com.    IN NS   ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.         IN      A       1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-
-; response to DNSKEY priming query
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN DNSKEY
-SECTION ANSWER
-example.com.	3600	IN	DNSKEY	256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}
-example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 512b}
-example.com.	3600	IN	RRSIG	DNSKEY 3 2 3600 20070926134150 20070829134150 2854 example.com. AKIIYDOGHogglFqJK94ZtOnF7EfGikgAyloMNRSMCrQgFaFkmcOyjrc= ;{id = 2854}
-example.com.	3600	IN	RRSIG	DNSKEY 5 2 3600 20070926134150 20070829134150 30899 example.com. J55fsz1GGMnngc4r50xvXDUdaVMlfcLKLVsfMhwNLF+ERac5XV/lLRAc/aSER+qQdsSo0CrjYjy1wat7YQpDAA== ;{id = 30899}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-; response to query of interest
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A	10.20.30.40
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
-www.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-www.example.com.        3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-RANGE_END
-
-STEP 1 QUERY
-ENTRY_BEGIN
-REPLY RD DO
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-
-; recursion happens here.
-STEP 10 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all
-REPLY QR RD RA AD DO NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A	10.20.30.40
-www.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
-www.example.com.        3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-SCENARIO_END

external/unbound/testdata/val_ta_algo_missing_dp.rpl

@@ -1,185 +0,0 @@
-; config options
-; The island of trust is at example.com
-server:
-	trust-anchor: "example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}"
-	trust-anchor: "example.com.	3600	IN	DS	30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512"
-	trust-anchor: "example.com.	3600	IN	DS	30899 7 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512"
-	val-override-date: "20070916134226"
-	target-fetch-policy: "0 0 0 0 0"
-	harden-algo-downgrade: no
-
-stub-zone:
-	name: "."
-	stub-addr: 193.0.14.129 	# K.ROOT-SERVERS.NET.
-CONFIG_END
-
-SCENARIO_BEGIN Test validator with multiple algorithm missing one
-
-; K.ROOT-SERVERS.NET.
-RANGE_BEGIN 0 100
-	ADDRESS 193.0.14.129 
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-. IN NS
-SECTION ANSWER
-. IN NS	K.ROOT-SERVERS.NET.
-SECTION ADDITIONAL
-K.ROOT-SERVERS.NET.	IN	A	193.0.14.129
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION AUTHORITY
-com.	IN NS	a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.	IN 	A	192.5.6.30
-ENTRY_END
-RANGE_END
-
-; a.gtld-servers.net.
-RANGE_BEGIN 0 100
-	ADDRESS 192.5.6.30
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-com. IN NS
-SECTION ANSWER
-com.    IN NS   a.gtld-servers.net.
-SECTION ADDITIONAL
-a.gtld-servers.net.     IN      A       192.5.6.30
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ENTRY_END
-RANGE_END
-
-; ns.example.com.
-RANGE_BEGIN 0 100
-	ADDRESS 1.2.3.4
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN NS
-SECTION ANSWER
-example.com.    IN NS   ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.         IN      A       1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR AA NOERROR
-SECTION QUESTION
-ns.example.com. IN AAAA
-SECTION ANSWER
-SECTION AUTHORITY
-example.com.    IN NS   ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.         IN      A       1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-
-; response to DNSKEY priming query
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-example.com. IN DNSKEY
-SECTION ANSWER
-example.com.	3600	IN	DNSKEY	256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}
-example.com.	3600	IN	DNSKEY	256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 512b}
-example.com.	3600	IN	RRSIG	DNSKEY 3 2 3600 20070926134150 20070829134150 2854 example.com. AKIIYDOGHogglFqJK94ZtOnF7EfGikgAyloMNRSMCrQgFaFkmcOyjrc= ;{id = 2854}
-example.com.	3600	IN	RRSIG	DNSKEY 5 2 3600 20070926134150 20070829134150 30899 example.com. J55fsz1GGMnngc4r50xvXDUdaVMlfcLKLVsfMhwNLF+ERac5XV/lLRAc/aSER+qQdsSo0CrjYjy1wat7YQpDAA== ;{id = 30899}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-; response to query of interest
-ENTRY_BEGIN
-MATCH opcode qtype qname
-ADJUST copy_id
-REPLY QR NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com. IN A	10.20.30.40
-ns.example.com. 3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
-www.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
-SECTION AUTHORITY
-example.com.	IN NS	ns.example.com.
-example.com.    3600    IN      RRSIG   NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-SECTION ADDITIONAL
-ns.example.com.		IN 	A	1.2.3.4
-www.example.com.        3600    IN      RRSIG   A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-RANGE_END
-
-STEP 1 QUERY
-ENTRY_BEGIN
-REPLY RD DO
-SECTION QUESTION
-www.example.com. IN A
-ENTRY_END
-
-; recursion happens here.
-STEP 10 CHECK_ANSWER
-ENTRY_BEGIN
-MATCH all
-REPLY QR RD RA AD DO NOERROR
-SECTION QUESTION
-www.example.com. IN A
-SECTION ANSWER
-www.example.com.	3600	IN	A	10.20.30.40
-www.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
-www.example.com.	3600	IN	RRSIG	A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
-
-SECTION AUTHORITY
-example.com.	3600	IN	NS	ns.example.com.
-example.com.	3600	IN	RRSIG	NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
-example.com.	3600	IN	RRSIG	NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
-
-SECTION ADDITIONAL
-ns.example.com.	3600	IN	A	1.2.3.4
-ns.example.com.	3600	IN	RRSIG	A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
-ns.example.com.	3600	IN	RRSIG	A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
-ENTRY_END
-
-SCENARIO_END

external/unbound/util/alloc.c

@@ -364,6 +364,9 @@ void *unbound_stat_malloc(size_t size)
 #ifdef calloc
 #undef calloc
 #endif
+#ifndef INT_MAX
+#define INT_MAX (((int)-1)>>1)
+#endif
 /** calloc with stats */
 void *unbound_stat_calloc(size_t nmemb, size_t size)
 {

external/unbound/util/config_file.c

@@ -70,6 +70,8 @@
 uid_t cfg_uid = (uid_t)-1;
 /** from cfg username, after daemonise setup performed */
 gid_t cfg_gid = (gid_t)-1;
+/** for debug allow small timeout values for fast rollovers */
+int autr_permit_small_holddown = 0;
 
 /** global config during parsing */
 struct config_parser_state* cfg_parser = 0;
@@ -200,6 +202,7 @@ config_create(void)
 	cfg->add_holddown = 30*24*3600;
 	cfg->del_holddown = 30*24*3600;
 	cfg->keep_missing = 366*24*3600; /* one year plus a little leeway */
+	cfg->permit_small_holddown = 0;
 	cfg->key_cache_size = 4 * 1024 * 1024;
 	cfg->key_cache_slabs = 4;
 	cfg->neg_cache_size = 1 * 1024 * 1024;
@@ -444,6 +447,9 @@ int config_set_option(struct config_file* cfg, const char* opt,
 	else S_UNSIGNED_OR_ZERO("add-holddown:", add_holddown)
 	else S_UNSIGNED_OR_ZERO("del-holddown:", del_holddown)
 	else S_UNSIGNED_OR_ZERO("keep-missing:", keep_missing)
+	else if(strcmp(opt, "permit-small-holddown:") == 0)
+	{ IS_YES_OR_NO; cfg->permit_small_holddown = (strcmp(val, "yes") == 0);
+	  autr_permit_small_holddown = cfg->permit_small_holddown; }
 	else S_MEMSIZE("key-cache-size:", key_cache_size)
 	else S_POW2("key-cache-slabs:", key_cache_slabs)
 	else S_MEMSIZE("neg-cache-size:", neg_cache_size)
@@ -705,6 +711,7 @@ config_get_option(struct config_file* cfg, const char* opt,
 	else O_UNS(opt, "add-holddown", add_holddown)
 	else O_UNS(opt, "del-holddown", del_holddown)
 	else O_UNS(opt, "keep-missing", keep_missing)
+	else O_YNO(opt, "permit-small-holddown", permit_small_holddown)
 	else O_MEM(opt, "key-cache-size", key_cache_size)
 	else O_DEC(opt, "key-cache-slabs", key_cache_slabs)
 	else O_MEM(opt, "neg-cache-size", neg_cache_size)
@@ -1243,6 +1250,7 @@ config_apply(struct config_file* config)
 	MINIMAL_RESPONSES = config->minimal_responses;
 	RRSET_ROUNDROBIN = config->rrset_roundrobin;
 	log_set_time_asc(config->log_time_ascii);
+	autr_permit_small_holddown = config->permit_small_holddown;
 }
 
 void config_lookup_uid(struct config_file* cfg)

external/unbound/util/config_file.h

@@ -269,6 +269,8 @@ struct config_file {
 	unsigned int del_holddown;
 	/** autotrust keep_missing time, in seconds. 0 is forever. */
 	unsigned int keep_missing;
+	/** permit small holddown values, allowing 5011 rollover very fast */
+	int permit_small_holddown;
 
 	/** size of the key cache */
 	size_t key_cache_size;
@@ -368,6 +370,8 @@ struct config_file {
 extern uid_t cfg_uid;
 /** from cfg username, after daemonise setup performed */
 extern gid_t cfg_gid;
+/** debug and enable small timeouts */
+extern int autr_permit_small_holddown;
 
 /**
  * Stub config options

external/unbound/util/configlexer.lex

@@ -306,6 +306,7 @@ val-nsec3-keysize-iterations{COLON}	{
 add-holddown{COLON}		{ YDVAR(1, VAR_ADD_HOLDDOWN) }
 del-holddown{COLON}		{ YDVAR(1, VAR_DEL_HOLDDOWN) }
 keep-missing{COLON}		{ YDVAR(1, VAR_KEEP_MISSING) }
+permit-small-holddown{COLON}	{ YDVAR(1, VAR_PERMIT_SMALL_HOLDDOWN) }
 use-syslog{COLON}		{ YDVAR(1, VAR_USE_SYSLOG) }
 log-time-ascii{COLON}		{ YDVAR(1, VAR_LOG_TIME_ASCII) }
 log-queries{COLON}		{ YDVAR(1, VAR_LOG_QUERIES) }

external/unbound/util/configparser.h

@@ -203,7 +203,8 @@ extern int yydebug;
      VAR_RATELIMIT_BELOW_DOMAIN = 412,
      VAR_RATELIMIT_FACTOR = 413,
      VAR_CAPS_WHITELIST = 414,
-     VAR_CACHE_MAX_NEGATIVE_TTL = 415
+     VAR_CACHE_MAX_NEGATIVE_TTL = 415,
+     VAR_PERMIT_SMALL_HOLDDOWN = 416
    };
 #endif
 /* Tokens.  */
@@ -365,6 +366,7 @@ extern int yydebug;
 #define VAR_RATELIMIT_FACTOR 413
 #define VAR_CAPS_WHITELIST 414
 #define VAR_CACHE_MAX_NEGATIVE_TTL 415
+#define VAR_PERMIT_SMALL_HOLDDOWN 416
 
 
 
@@ -378,7 +380,7 @@ typedef union YYSTYPE
 
 
 /* Line 2058 of yacc.c  */
-#line 382 "util/configparser.h"
+#line 384 "util/configparser.h"
 } YYSTYPE;
 # define YYSTYPE_IS_TRIVIAL 1
 # define yystype YYSTYPE /* obsolescent; will be withdrawn */

external/unbound/util/configparser.y

@@ -121,7 +121,7 @@ extern struct config_parser_state* cfg_parser;
 %token VAR_HARDEN_ALGO_DOWNGRADE VAR_IP_TRANSPARENT
 %token VAR_RATELIMIT VAR_RATELIMIT_SLABS VAR_RATELIMIT_SIZE
 %token VAR_RATELIMIT_FOR_DOMAIN VAR_RATELIMIT_BELOW_DOMAIN VAR_RATELIMIT_FACTOR
-%token VAR_CAPS_WHITELIST VAR_CACHE_MAX_NEGATIVE_TTL
+%token VAR_CAPS_WHITELIST VAR_CACHE_MAX_NEGATIVE_TTL VAR_PERMIT_SMALL_HOLDDOWN
 
 %%
 toplevelvars: /* empty */ | toplevelvars toplevelvar ;
@@ -185,7 +185,8 @@ content_server: server_num_threads | server_verbosity | server_port |
 	server_ip_transparent | server_ratelimit | server_ratelimit_slabs |
 	server_ratelimit_size | server_ratelimit_for_domain |
 	server_ratelimit_below_domain | server_ratelimit_factor |
-	server_caps_whitelist | server_cache_max_negative_ttl
+	server_caps_whitelist | server_cache_max_negative_ttl |
+	server_permit_small_holddown
 	;
 stubstart: VAR_STUB_ZONE
 	{
@@ -1125,6 +1126,15 @@ server_keep_missing: VAR_KEEP_MISSING STRING_ARG
 		free($2);
 	}
 	;
+server_permit_small_holddown: VAR_PERMIT_SMALL_HOLDDOWN STRING_ARG
+	{
+		OUTYY(("P(server_permit_small_holddown:%s)\n", $2));
+		if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0)
+			yyerror("expected yes or no.");
+		else cfg_parser->cfg->permit_small_holddown =
+			(strcmp($2, "yes")==0);
+		free($2);
+	}
 server_key_cache_size: VAR_KEY_CACHE_SIZE STRING_ARG
 	{
 		OUTYY(("P(server_key_cache_size:%s)\n", $2));

external/unbound/util/data/msgencode.c

@@ -283,7 +283,7 @@ compress_owner(struct ub_packed_rrset_key* key, sldns_buffer* pkt,
 	size_t owner_pos, uint16_t* owner_ptr, int owner_labs)
 {
 	struct compress_tree_node* p;
-	struct compress_tree_node** insertpt;
+	struct compress_tree_node** insertpt = NULL;
 	if(!*owner_ptr) {
 		/* compress first time dname */
 		if((p = compress_tree_lookup(tree, key->rk.dname, 

external/unbound/util/data/msgreply.c

@@ -822,13 +822,13 @@ log_query_info(enum verbosity_value v, const char* str,
 }
 
 int
-reply_check_cname_chain(struct reply_info* rep) 
+reply_check_cname_chain(struct query_info* qinfo, struct reply_info* rep) 
 {
 	/* check only answer section rrs for matching cname chain.
 	 * the cache may return changed rdata, but owner names are untouched.*/
 	size_t i;
-	uint8_t* sname = rep->rrsets[0]->rk.dname;
+	uint8_t* sname = qinfo->qname;
-	size_t snamelen = rep->rrsets[0]->rk.dname_len;
+	size_t snamelen = qinfo->qname_len;
 	for(i=0; i<rep->an_numrrsets; i++) {
 		uint16_t t = ntohs(rep->rrsets[i]->rk.type);
 		if(t == LDNS_RR_TYPE_DNAME)

external/unbound/util/data/msgreply.h

@@ -359,10 +359,11 @@ uint8_t* reply_find_final_cname_target(struct query_info* qinfo,
 
 /**
  * Check if cname chain in cached reply is still valid.
+ * @param qinfo: query info with query name.
  * @param rep: reply to check.
  * @return: true if valid, false if invalid.
  */
-int reply_check_cname_chain(struct reply_info* rep);
+int reply_check_cname_chain(struct query_info* qinfo, struct reply_info* rep);
 
 /**
  * Check security status of all RRs in the message.

external/unbound/util/iana_ports.inc

@@ -1066,7 +1066,6 @@
 1404,
 1405,
 1406,
-1407,
 1408,
 1409,
 1410,
@@ -4667,6 +4666,7 @@
 7725,
 7726,
 7727,
+7728,
 7734,
 7738,
 7741,
@@ -4781,6 +4781,7 @@
 8301,
 8320,
 8321,
+8322,
 8351,
 8376,
 8377,
@@ -4788,6 +4789,7 @@
 8379,
 8380,
 8383,
+8384,
 8400,
 8401,
 8402,
@@ -4804,6 +4806,7 @@
 8474,
 8500,
 8501,
+8503,
 8554,
 8555,
 8567,
@@ -5034,6 +5037,7 @@
 10200,
 10201,
 10252,
+10253,
 10260,
 10288,
 10439,
@@ -5168,6 +5172,8 @@
 17220,
 17221,
 17222,
+17224,
+17225,
 17234,
 17235,
 17500,
@@ -5380,6 +5386,7 @@
 40843,
 40853,
 41111,
+41230,
 41794,
 41795,
 42508,

external/unbound/util/net_help.c

@@ -629,9 +629,9 @@ void* listen_sslctx_create(char* key, char* pem, char* verifypem)
 		SSL_CTX_free(ctx);
 		return NULL;
 	}
-	if(!SSL_CTX_use_certificate_file(ctx, pem, SSL_FILETYPE_PEM)) {
+	if(!SSL_CTX_use_certificate_chain_file(ctx, pem)) {
 		log_err("error for cert file: %s", pem);
-		log_crypto_err("error in SSL_CTX use_certificate_file");
+		log_crypto_err("error in SSL_CTX use_certificate_chain_file");
 		SSL_CTX_free(ctx);
 		return NULL;
 	}
@@ -647,6 +647,23 @@ void* listen_sslctx_create(char* key, char* pem, char* verifypem)
 		SSL_CTX_free(ctx);
 		return NULL;
 	}
+#if HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
+	if(!SSL_CTX_set_ecdh_auto(ctx,1)) {
+		log_crypto_err("Error in SSL_CTX_ecdh_auto, not enabling ECDHE");
+	}
+#elif defined(USE_ECDSA)
+	if(1) {
+		EC_KEY *ecdh = EC_KEY_new_by_curve_name (NID_X9_62_prime256v1);
+		if (!ecdh) {
+			log_crypto_err("could not find p256, not enabling ECDHE");
+		} else {
+			if (1 != SSL_CTX_set_tmp_ecdh (ctx, ecdh)) {
+				log_crypto_err("Error in SSL_CTX_set_tmp_ecdh, not enabling ECDHE");
+			}
+			EC_KEY_free (ecdh);
+		}
+	}
+#endif
 
 	if(verifypem && verifypem[0]) {
 		if(!SSL_CTX_load_verify_locations(ctx, verifypem, NULL)) {
@@ -684,7 +701,7 @@ void* connect_sslctx_create(char* key, char* pem, char* verifypem)
 		return NULL;
 	}
 	if(key && key[0]) {
-		if(!SSL_CTX_use_certificate_file(ctx, pem, SSL_FILETYPE_PEM)) {
+		if(!SSL_CTX_use_certificate_chain_file(ctx, pem)) {
 			log_err("error in client certificate %s", pem);
 			log_crypto_err("error in certificate file");
 			SSL_CTX_free(ctx);

external/unbound/validator/autotrust.c

@@ -1225,7 +1225,7 @@ verify_dnskey(struct module_env* env, struct val_env* ve,
 {
 	char* reason = NULL;
 	uint8_t sigalg[ALGO_NEEDS_MAX+1];
-	int downprot = 1;
+	int downprot = env->cfg->harden_algo_downgrade;
 	enum sec_status sec = val_verify_DNSKEY_with_TA(env, ve, rrset,
 		tp->ds_rrset, tp->dnskey_rrset, downprot?sigalg:NULL, &reason);
 	/* sigalg is ignored, it returns algorithms signalled to exist, but
@@ -1447,9 +1447,11 @@ set_tp_times(struct trust_anchor* tp, time_t rrsig_exp_interval,
 	if(rrsig_exp_interval/2 < x)
 		x = rrsig_exp_interval/2;
 	/* MAX(1hr, x) */
-	if(x < 3600)
+	if(!autr_permit_small_holddown) {
-		tp->autr->query_interval = 3600;
+		if(x < 3600)
-	else	tp->autr->query_interval = x;
+			tp->autr->query_interval = 3600;
+		else	tp->autr->query_interval = x;
+	}	else    tp->autr->query_interval = x;
 
 	/* x= MIN(1day, ttl/10, expire/10) */
 	x = 24 * 3600;
@@ -1458,9 +1460,11 @@ set_tp_times(struct trust_anchor* tp, time_t rrsig_exp_interval,
 	if(rrsig_exp_interval/10 < x)
 		x = rrsig_exp_interval/10;
 	/* MAX(1hr, x) */
-	if(x < 3600)
+	if(!autr_permit_small_holddown) {
-		tp->autr->retry_time = 3600;
+		if(x < 3600)
-	else	tp->autr->retry_time = x;
+			tp->autr->retry_time = 3600;
+		else	tp->autr->retry_time = x;
+	}	else    tp->autr->retry_time = x;
 
 	if(qi != tp->autr->query_interval || rt != tp->autr->retry_time) {
 		*changed = 1;
@@ -1959,8 +1963,12 @@ calc_next_probe(struct module_env* env, time_t wait)
 {
 	/* make it random, 90-100% */
 	time_t rnd, rest;
-	if(wait < 3600)
+	if(!autr_permit_small_holddown) {
-		wait = 3600;
+		if(wait < 3600)
+			wait = 3600;
+	} else {
+		if(wait == 0) wait = 1;
+	}
 	rnd = wait/10;
 	rest = wait-rnd;
 	rnd = (time_t)ub_random_max(env->rnd, (long int)rnd);
@@ -2349,6 +2357,8 @@ todo_probe(struct module_env* env, time_t* next)
 	if( (el=rbtree_first(&env->anchors->autr->probe)) == RBTREE_NULL) {
 		/* in case of revoked anchors */
 		lock_basic_unlock(&env->anchors->lock);
+		/* signal that there are no anchors to probe */
+		*next = 0;
 		return NULL;
 	}
 	tp = (struct trust_anchor*)el->key;
@@ -2378,6 +2388,7 @@ autr_probe_timer(struct module_env* env)
 	struct trust_anchor* tp;
 	time_t next_probe = 3600;
 	int num = 0;
+	if(autr_permit_small_holddown) next_probe = 1;
 	verbose(VERB_ALGO, "autotrust probe timer callback");
 	/* while there are still anchors to probe */
 	while( (tp = todo_probe(env, &next_probe)) ) {
@@ -2386,7 +2397,7 @@ autr_probe_timer(struct module_env* env)
 		num++;
 	}
 	regional_free_all(env->scratch);
-	if(num == 0)
+	if(next_probe == 0)
 		return 0; /* no trust points to probe */
 	verbose(VERB_ALGO, "autotrust probe timer %d callbacks done", num);
 	return next_probe;

external/unbound/validator/validator.c

@@ -2769,7 +2769,7 @@ process_dnskey_response(struct module_qstate* qstate, struct val_qstate* vq,
 		vq->state = VAL_VALIDATE_STATE;
 		return;
 	}
-	downprot = 1;
+	downprot = qstate->env->cfg->harden_algo_downgrade;
 	vq->key_entry = val_verify_new_DNSKEYs(qstate->region, qstate->env,
 		ve, dnskey, vq->ds_rrset, downprot, &reason);
 

src/CMakeLists.txt

@@ -106,3 +106,7 @@ add_subdirectory(daemonizer)
 add_subdirectory(daemon)
 
 add_subdirectory(blockchain_utilities)
+
+if(PER_BLOCK_CHECKPOINT)
+	add_subdirectory(blocks)
+endif()

src/blockchain_db/CMakeLists.txt

@@ -31,7 +31,7 @@ set(blockchain_db_sources
   lmdb/db_lmdb.cpp
   )
 
-if (NOT STATIC)
+if (BERKELEY_DB)
   set(blockchain_db_sources
   ${blockchain_db_sources}
   berkeleydb/db_bdb.cpp
@@ -46,7 +46,7 @@ set(blockchain_db_private_headers
   lmdb/db_lmdb.h
   )
 
-if (NOT STATIC)
+if (BERKELEY_DB)
   set(blockchain_db_private_headers
   ${blockchain_db_private_headers}
   berkeleydb/db_bdb.h

src/blockchain_db/berkeleydb/db_bdb.h

@@ -30,6 +30,11 @@
 #include "blockchain_db/blockchain_db.h"
 #include "cryptonote_protocol/blobdatatype.h" // for type blobdata
 
+#include <unordered_map>
+
+// ND: Enables multi-threaded bulk reads for when getting indices.
+//     TODO: Disabled for now, as it doesn't seem to provide noticeable improvements (??. Reason: TBD.
+// #define BDB_BULK_CAN_THREAD
 namespace cryptonote
 {
 
@@ -83,10 +88,145 @@ struct bdb_txn_safe
   {
     return &m_txn;
   }
-
+private:
   DbTxn* m_txn;
 };
 
+// ND: Class to handle buffer management when doing bulk queries
+// (DB_MULTIPLE). Allocates buffers then handles thread queuing
+// so a fixed set of buffers can be used (instead of allocating
+// every time a bulk query is needed).
+template <typename T>
+class bdb_safe_buffer
+{
+	// limit the number of buffers to 8
+	const size_t MaxAllowedBuffers = 8;
+public:
+    bdb_safe_buffer(size_t num_buffers, size_t count)
+    {
+    	if(num_buffers > MaxAllowedBuffers)
+    		num_buffers = MaxAllowedBuffers;
+    	
+        set_count(num_buffers);
+        for (size_t i = 0; i < num_buffers; i++)
+            m_buffers.push_back((T) malloc(sizeof(T) * count));
+        m_buffer_count = count;
+    }
+
+    ~bdb_safe_buffer()
+    {
+        for (size_t i = 0; i < m_buffers.size(); i++)
+        {
+            if (m_buffers[i])
+            {
+                free(m_buffers[i]);
+                m_buffers[i] = nullptr;
+            }
+        }
+
+        m_buffers.resize(0);
+    }
+
+    T acquire_buffer()
+    {
+        std::unique_lock<std::mutex> lock(m_lock);
+        m_cv.wait(lock, [&]{ return m_count > 0; });
+
+        --m_count;
+        size_t index = -1;
+        for (size_t i = 0; i < m_open_slot.size(); i++)
+        {
+            if (m_open_slot[i])
+            {
+                m_open_slot[i] = false;
+                index = i;
+                break;
+            }
+        }
+
+        assert(index >= 0);
+
+        T buffer = m_buffers[index];
+        m_buffer_map.emplace(buffer, index);
+        return buffer;
+    }
+
+    void release_buffer(T buffer)
+    {
+        std::unique_lock<std::mutex> lock(m_lock);
+
+        assert(buffer != nullptr);
+        auto it = m_buffer_map.find(buffer);
+        if (it != m_buffer_map.end())
+        {
+            auto index = it->second;
+
+            assert(index < m_open_slot.size());
+            assert(m_open_slot[index] == false);
+            assert(m_count < m_open_slot.size());
+
+            ++m_count;
+            m_open_slot[index] = true;
+            m_buffer_map.erase(it);
+            m_cv.notify_one();
+        }
+    }
+
+    size_t get_buffer_size() const
+    {
+        return m_buffer_count * sizeof(T);
+    }
+
+    size_t get_buffer_count() const
+    {
+        return m_buffer_count;
+    }
+
+    typedef T type;
+
+private:
+    void set_count(size_t count)
+    {
+        assert(count > 0);
+        m_open_slot.resize(count, true);
+        m_count = count;
+    }
+
+    std::vector<T> m_buffers;
+    std::unordered_map<T, size_t> m_buffer_map;
+
+    std::condition_variable m_cv;
+    std::vector<bool> m_open_slot;
+    size_t m_count;
+    std::mutex m_lock;
+
+    size_t m_buffer_count;
+};
+
+template <typename T>
+class bdb_safe_buffer_autolock
+{
+public:
+    bdb_safe_buffer_autolock(T &safe_buffer, typename T::type &buffer) :
+        m_safe_buffer(safe_buffer), m_buffer(nullptr)
+    {
+        m_buffer = m_safe_buffer.acquire_buffer();
+        buffer = m_buffer;
+    }
+
+    ~bdb_safe_buffer_autolock()
+    {
+        if (m_buffer != nullptr)
+        {
+            m_safe_buffer.release_buffer(m_buffer);
+            m_buffer = nullptr;
+        }
+    }
+private:
+    T &m_safe_buffer;
+    typename T::type m_buffer;
+};
+
 class BlockchainBDB : public BlockchainDB
 {
 public:
@@ -155,12 +295,11 @@ public:
 
   virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
 
-  virtual uint64_t get_random_output(const uint64_t& amount) const;
-
   virtual uint64_t get_num_outputs(const uint64_t& amount) const;
 
-  virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const;
+  virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
-
+  virtual output_data_t get_output_key(const uint64_t& global_index) const;
+  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs);
   virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const;
 
   /**
@@ -175,8 +314,11 @@ public:
   tx_out get_output(const uint64_t& index) const;
 
   virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
+  virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
+          std::vector<tx_out_index> &tx_out_indices) const;
 
-  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
+  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
+  virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
 
   virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
   virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
@@ -191,13 +333,19 @@ public:
                             );
 
   virtual void set_batch_transactions(bool batch_transactions);
-  virtual void batch_start();
+  virtual void batch_start(uint64_t batch_num_blocks=0);
   virtual void batch_commit();
   virtual void batch_stop();
   virtual void batch_abort();
 
   virtual void pop_block(block& blk, std::vector<transaction>& txs);
 
+#if defined(BDB_BULK_CAN_THREAD)
+  virtual bool can_thread_bulk_indices() const { return true; }
+#else
+  virtual bool can_thread_bulk_indices() const { return false; }
+#endif
+
 private:
   virtual void add_block( const block& blk
                 , const size_t& block_size
@@ -212,7 +360,7 @@ private:
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index);
+  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
 
   virtual void remove_output(const tx_out& tx_output);
 
@@ -225,6 +373,7 @@ private:
 
   virtual void remove_spent_key(const crypto::key_image& k_image);
 
+  void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices);
   /**
    * @brief convert a tx output to a blob for storage
    *
@@ -251,9 +400,13 @@ private:
    *
    * @return the global index of the desired output
    */
-  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const;
+  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
-
+  void checkpoint_worker() const;
   void check_open() const;
+  bool m_run_checkpoint;
+  std::unique_ptr<boost::thread> m_checkpoint_thread;
+  typedef bdb_safe_buffer<void *> bdb_safe_buffer_t;
+  bdb_safe_buffer_t m_buffer;
 
   DbEnv* m_env;
 

src/blockchain_db/blockchain_db.cpp

@@ -56,24 +56,37 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
     tx_hash = *tx_hash_ptr;
   }
 
+  for (const txin_v& tx_input : tx.vin)
+  {
+    if (tx_input.type() == typeid(txin_to_key))
+    {
+      add_spent_key(boost::get<txin_to_key>(tx_input).k_image);
+    }
+    else if (tx_input.type() == typeid(txin_gen))
+    {
+      /* nothing to do here */
+    }
+    else
+    {
+      LOG_PRINT_L1("Unsupported input type, removing key images and aborting transaction addition");
+      for (const txin_v& tx_input : tx.vin)
+      {
+        if (tx_input.type() == typeid(txin_to_key))
+        {
+          remove_spent_key(boost::get<txin_to_key>(tx_input).k_image);
+        }
+      }
+      return;
+    }
+  }
+
   add_transaction_data(blk_hash, tx, tx_hash);
 
   // iterate tx.vout using indices instead of C++11 foreach syntax because
   // we need the index
-  if (tx.vout.size() != 0)  // it may be technically possible for a tx to have no outputs
+  for (uint64_t i = 0; i < tx.vout.size(); ++i)
   {
-    for (uint64_t i = 0; i < tx.vout.size(); ++i)
+    add_output(tx_hash, tx.vout[i], i, tx.unlock_time);
-    {
-      add_output(tx_hash, tx.vout[i], i);
-    }
-
-    for (const txin_v& tx_input : tx.vin)
-    {
-      if (tx_input.type() == typeid(txin_to_key))
-      {
-        add_spent_key(boost::get<txin_to_key>(tx_input).k_image);
-      }
-    }
   }
 }
 

src/blockchain_db/blockchain_db.h

@@ -104,7 +104,6 @@
  *   height      get_tx_block_height(hash)
  *
  * Outputs:
- *   index       get_random_output(amount)
  *   uint64_t    get_num_outputs(amount)
  *   pub_key     get_output_key(amount, index)
  *   tx_out      get_output(tx_hash, index)
@@ -138,6 +137,15 @@ namespace cryptonote
 // typedef for convenience
 typedef std::pair<crypto::hash, uint64_t> tx_out_index;
 
+#pragma pack(push, 1)
+struct output_data_t
+{
+	crypto::public_key pubkey;
+	uint64_t unlock_time;
+	uint64_t height;
+};
+#pragma pack(pop)
+	
 /***********************************
  * Exception Definitions
  ***********************************/
@@ -279,7 +287,7 @@ private:
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0;
 
   // tells the subclass to store an output
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0;
+  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
 
   // tells the subclass to remove an output
   virtual void remove_output(const tx_out& tx_output) = 0;
@@ -313,7 +321,7 @@ protected:
 
   mutable uint64_t time_tx_exists = 0;
   uint64_t time_commit1 = 0;
-
+  bool m_auto_remove_logs = true;
 
 public:
 
@@ -357,7 +365,7 @@ public:
   // release db lock
   virtual void unlock() = 0;
 
-  virtual void batch_start() = 0;
+  virtual void batch_start(uint64_t batch_num_blocks=0) = 0;
   virtual void batch_stop() = 0;
   virtual void set_batch_transactions(bool) = 0;
 
@@ -454,14 +462,12 @@ public:
   // returns height of block that contains transaction with hash <h>
   virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0;
 
-  // return global output index of a random output of amount <amount>
-  virtual uint64_t get_random_output(const uint64_t& amount) const = 0;
-
   // returns the total number of outputs of amount <amount>
   virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0;
 
   // return public key for output with global output amount <amount> and index <index>
-  virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0;
+  virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0;
+  virtual output_data_t get_output_key(const uint64_t& global_index) const = 0;
 
   // returns the output indexed by <index> in the transaction with hash <h>
   virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0;
@@ -471,7 +477,11 @@ public:
 
   // returns the transaction-local reference for the output with <amount> at <index>
   // return type is pair of tx hash and index
-  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0;
+  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) = 0;
+  virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) = 0;
+  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) = 0;
+  
+  virtual bool can_thread_bulk_indices() const = 0;
 
   // return a vector of indices corresponding to the global output index for
   // each output in the transaction with hash <h>
@@ -483,7 +493,10 @@ public:
   // returns true if key image <img> is present in spent key images storage
   virtual bool has_key_image(const crypto::key_image& img) const = 0;
 
+  void set_auto_remove_logs(bool auto_remove) { m_auto_remove_logs = auto_remove; }
+
   bool m_open;
+  mutable epee::critical_section m_synchronization_lock;
 };  // class BlockchainDB
 
 

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -28,8 +28,10 @@
 #include "db_lmdb.h"
 
 #include <boost/filesystem.hpp>
+#include <boost/format.hpp>
 #include <memory>  // std::unique_ptr
 #include <cstring>  // memcpy
+#include <random>
 
 #include "cryptonote_core/cryptonote_format_utils.h"
 #include "crypto/crypto.h"
@@ -65,10 +67,19 @@ struct lmdb_cur
     done = false;
   }
 
-  ~lmdb_cur() { close(); }
+	~lmdb_cur()
+	{
+		close();
+	}
 
-  operator MDB_cursor*() { return m_cur; }
+	operator MDB_cursor*()
-  operator MDB_cursor**() { return &m_cur; }
+	{
+		return m_cur;
+	}
+	operator MDB_cursor**()
+	{
+		return &m_cur;
+	}
 
   void close()
   {
@@ -87,7 +98,8 @@ private:
 template<typename T>
 struct MDB_val_copy: public MDB_val
 {
-  MDB_val_copy(const T &t): t_copy(t)
+	MDB_val_copy(const T &t) :
+			t_copy(t)
   {
     mv_size = sizeof (T);
     mv_data = &t_copy;
@@ -99,7 +111,8 @@ private:
 template<>
 struct MDB_val_copy<cryptonote::blobdata>: public MDB_val
 {
-  MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()])
+	MDB_val_copy(const cryptonote::blobdata &bd) :
+			data(new char[bd.size()])
   {
     memcpy(data.get(), bd.data(), bd.size());
     mv_size = bd.size();
@@ -109,7 +122,8 @@ private:
   std::unique_ptr<char[]> data;
 };
 
-auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) {
+auto compare_uint64 = [](const MDB_val *a, const MDB_val *b)
+{
   const uint64_t va = *(const uint64_t*)a->mv_data;
   const uint64_t vb = *(const uint64_t*)b->mv_data;
   if (va < vb) return -1;
@@ -117,6 +131,20 @@ auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) {
   else return 1;
 };
 
+int compare_hash32(const MDB_val *a, const MDB_val *b)
+{
+	uint32_t *va = (uint32_t*) a->mv_data;
+	uint32_t *vb = (uint32_t*) b->mv_data;
+	for (int n = 7; n >= 0; n--)
+	{
+		if (va[n] == vb[n])
+			continue;
+		return va[n] < vb[n] ? -1 : 1;
+	}
+
+	return 0;
+}
+
 const char* const LMDB_BLOCKS = "blocks";
 const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps";
 const char* const LMDB_BLOCK_HEIGHTS = "block_heights";
@@ -233,8 +261,29 @@ void mdb_txn_safe::allow_new_txns()
 
 
 
-void BlockchainLMDB::do_resize()
+void BlockchainLMDB::do_resize(uint64_t increase_size)
 {
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  CRITICAL_REGION_LOCAL(m_synchronization_lock);
+  const uint64_t add_size = 1LL << 30;
+
+  // check disk capacity
+  try
+  {
+	boost::filesystem::path path(m_folder);
+	boost::filesystem::space_info si = boost::filesystem::space(path);
+	if(si.available < add_size)
+	{
+	  LOG_PRINT_RED_L0("!! WARNING: Insufficient free space to extend database !!: " << si.available / 1LL << 20L);
+	  return;
+	}
+  }
+  catch(...)
+  {
+	// print something but proceed.
+	LOG_PRINT_YELLOW("Unable to query free disk space.", LOG_LEVEL_0);
+  }
+
   MDB_envinfo mei;
 
   mdb_env_info(m_env, &mei);
@@ -243,7 +292,14 @@ void BlockchainLMDB::do_resize()
 
   mdb_env_stat(m_env, &mst);
 
-  uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR;
+  // add 1Gb per resize, instead of doing a percentage increase
+  uint64_t new_mapsize = (double) mei.me_mapsize + add_size;
+
+  // If given, use increase_size intead of above way of resizing.
+  // This is currently used for increasing by an estimated size at start of new
+  // batch txn.
+  if (increase_size > 0)
+    new_mapsize = mei.me_mapsize + increase_size;
 
   new_mapsize += (new_mapsize % mst.ms_psize);
 
@@ -265,15 +321,16 @@ void BlockchainLMDB::do_resize()
 
   mdb_env_set_mapsize(m_env, new_mapsize);
 
-  LOG_PRINT_L0("LMDB Mapsize increased."
+  LOG_PRINT_GREEN("LMDB Mapsize increased." << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0);
-      << "  Old: " << mei.me_mapsize / (1024 * 1024) << "MiB"
-      << ", New: " << new_mapsize / (1024 * 1024) << "MiB");
 
   mdb_txn_safe::allow_new_txns();
 }
 
-bool BlockchainLMDB::need_resize() const
+// threshold_size is used for batch transactions
+bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
 {
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+#if defined(ENABLE_AUTO_RESIZE)
   MDB_envinfo mei;
 
   mdb_env_info(m_env, &mei);
@@ -282,21 +339,130 @@ bool BlockchainLMDB::need_resize() const
 
   mdb_env_stat(m_env, &mst);
 
+  // size_used doesn't include data yet to be committed, which can be
+  // significant size during batch transactions. For that, we estimate the size
+  // needed at the beginning of the batch transaction and pass in the
+  // additional size needed.
   uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
 
-  if ((double)size_used / mei.me_mapsize  > RESIZE_PERCENT)
+  LOG_PRINT_L1("DB map size:     " << mei.me_mapsize);
+  LOG_PRINT_L1("Space used:      " << size_used);
+  LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used);
+  LOG_PRINT_L1("Size threshold:  " << threshold_size);
+  float resize_percent_old = RESIZE_PERCENT;
+  LOG_PRINT_L1(boost::format("Percent used: %.04f  Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old);
+
+  if (threshold_size > 0)
   {
+    if (mei.me_mapsize - size_used < threshold_size)
+    {
+      LOG_PRINT_L1("Threshold met (size-based)");
+      return true;
+    }
+    else
+      return false;
+  }
+
+  std::mt19937 engine(std::random_device{}());
+  std::uniform_real_distribution<double> fdis(0.6, 0.9);
+  double resize_percent = fdis(engine);
+
+  if ((double)size_used / mei.me_mapsize  > resize_percent)
+  {
+    LOG_PRINT_L1("Threshold met (percent-based)");
     return true;
   }
   return false;
+#else
+	return false;
+#endif
 }
 
-void BlockchainLMDB::add_block( const block& blk
+void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
-              , const size_t& block_size
+{
-              , const difficulty_type& cumulative_difficulty
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-              , const uint64_t& coins_generated
+  LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
-              , const crypto::hash& blk_hash
+  const uint64_t min_increase_size = 128 * (1 << 20);
-              )
+  uint64_t threshold_size = 0;
+  uint64_t increase_size = 0;
+  if (batch_num_blocks > 0)
+  {
+    threshold_size = get_estimated_batch_size(batch_num_blocks);
+    LOG_PRINT_L1("calculated batch size: " << threshold_size);
+
+    // The increased DB size could be a multiple of threshold_size, a fixed
+    // size increase (> threshold_size), or other variations.
+    //
+    // Currently we use the greater of threshold size and a minimum size. The
+    // minimum size increase is used to avoid frequent resizes when the batch
+    // size is set to a very small numbers of blocks.
+    increase_size = (threshold_size > min_increase_size) ? threshold_size : min_increase_size;
+    LOG_PRINT_L1("increase size: " << increase_size);
+  }
+
+  // if threshold_size is 0 (i.e. number of blocks for batch not passed in), it
+  // will fall back to the percent-based threshold check instead of the
+  // size-based check
+  if (need_resize(threshold_size))
+  {
+    LOG_PRINT_L0("[batch] DB resize needed");
+    do_resize(increase_size);
+  }
+}
+
+uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) const
+{
+  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+  uint64_t threshold_size = 0;
+
+  // batch size estimate * batch safety factor = final size estimate
+  // Takes into account "reasonable" block size increases in batch.
+  float batch_safety_factor = 1.7f;
+  // estimate of stored block expanded from raw block, including denormalization and db overhead.
+  // Note that this probably doesn't grow linearly with block size.
+  float db_expand_factor = 4.5f;
+  uint64_t num_prev_blocks = 500;
+  // For resizing purposes, allow for at least 4k average block size.
+  uint64_t min_block_size = 4 * 1024;
+
+  uint64_t block_stop = 0;
+  if (m_height > 1)
+    block_stop = m_height - 1;
+  uint64_t block_start = 0;
+  if (block_stop >= num_prev_blocks)
+    block_start = block_stop - num_prev_blocks + 1;
+  uint32_t num_blocks_used = 0;
+  uint64_t total_block_size = 0;
+  LOG_PRINT_L1("[" << __func__ << "] " << "m_height: " << m_height << "  block_start: " << block_start << "  block_stop: " << block_stop);
+  size_t avg_block_size = 0;
+  if (m_height == 0)
+  {
+    LOG_PRINT_L1("No existing blocks to check for average block size");
+  }
+  else
+  {
+    for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
+    {
+      uint32_t block_size = get_block_size(block_num);
+      total_block_size += block_size;
+      // Track number of blocks being totalled here instead of assuming, in case
+      // some blocks were to be skipped for being outliers.
+      ++num_blocks_used;
+    }
+    avg_block_size = total_block_size / num_blocks_used;
+    LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
+  }
+  if (avg_block_size < min_block_size)
+    avg_block_size = min_block_size;
+  LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
+
+  threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
+  threshold_size = threshold_size * batch_safety_factor;
+  return threshold_size;
+}
+
+void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
+		const crypto::hash& blk_hash)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -447,7 +613,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
 
 }
 
-void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index)
+void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
@@ -476,10 +642,15 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou
 
   if (tx_output.target.type() == typeid(txout_to_key))
   {
-    MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key);
+		output_data_t od;
-    result = mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0);
+		od.pubkey = boost::get < txout_to_key > (tx_output.target).key;
-    if (result)
+		od.unlock_time = unlock_time;
-      throw0(DB_ERROR(std::string("Failed to add output pubkey to db transaction: ").append(mdb_strerror(result)).c_str()));
+		od.height = m_height;
+
+		MDB_val_copy<output_data_t> data(od);
+		//MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key);
+		if (mdb_put(*m_write_txn, m_output_keys, &k, &data, 0))
+			throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
   }
 
 
@@ -710,47 +881,17 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const
   return o;
 }
 
-uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const
+uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) 
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  check_open();
+	std::vector <uint64_t> offsets;
-
+	std::vector <uint64_t> global_indices;
-  mdb_txn_safe txn;
+	offsets.push_back(index);
-  if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+	get_output_global_indices(amount, offsets, global_indices);
-    throw0(DB_ERROR("Failed to create a transaction for the db"));
+	if (!global_indices.size())
-
-  lmdb_cur cur(txn, m_output_amounts);
-
-  MDB_val_copy<uint64_t> k(amount);
-  MDB_val v;
-
-  auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
-  if (result == MDB_NOTFOUND)
     throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
-  else if (result)
-    throw0(DB_ERROR("DB error attempting to get an output"));
 
-  size_t num_elems = 0;
+	return global_indices[0];
-  mdb_cursor_count(cur, &num_elems);
-  if (num_elems <= index)
-    throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
-
-  mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
-
-  for (uint64_t i = 0; i < index; ++i)
-  {
-    mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
-  }
-
-  mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
-
-  uint64_t glob_index = *(const uint64_t*)v.mv_data;
-
-  cur.close();
-
-  txn.commit();
-
-  return glob_index;
 }
 
 void BlockchainLMDB::check_open() const
@@ -805,8 +946,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   // check for existing LMDB files in base directory
   boost::filesystem::path old_files = direc.parent_path();
-  if (boost::filesystem::exists(old_files / "data.mdb") ||
+	if (boost::filesystem::exists(old_files / "data.mdb") || boost::filesystem::exists(old_files / "lock.mdb"))
-      boost::filesystem::exists(old_files / "lock.mdb"))
   {
     LOG_PRINT_L0("Found existing LMDB files in " << old_files.string());
     LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart");
@@ -872,7 +1012,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs");
   lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices");
-  lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
+	lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS,	MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
   lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys");
 
 /*************** not used, but kept for posterity
@@ -884,6 +1024,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
 
   mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
   mdb_set_dupsort(txn, m_tx_outputs, compare_uint64);
+  mdb_set_compare(txn, m_spent_keys, compare_hash32);
+  mdb_set_compare(txn, m_block_heights, compare_hash32);
+  mdb_set_compare(txn, m_txs, compare_hash32);
+  mdb_set_compare(txn, m_tx_unlocks, compare_hash32);
+  mdb_set_compare(txn, m_tx_heights, compare_hash32);
 
   // get and keep current height
   MDB_stat db_stats;
@@ -897,6 +1042,34 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
     throw0(DB_ERROR("Failed to query m_output_indices"));
   m_num_outputs = db_stats.ms_entries;
 
+	// ND: This "new" version of the lmdb database is incompatible with
+	// the previous version. Ensure that the output_keys database is
+	// sizeof(output_data_t) in length. Otherwise, inform user and
+	// terminate.
+	if(m_height > 0)
+	{
+	    MDB_val_copy<uint64_t> k(0);
+	    MDB_val v;
+	    auto get_result = mdb_get(txn, m_output_keys, &k, &v);
+	    if(get_result != MDB_SUCCESS)
+	    {
+	    	txn.abort();
+	    	m_open = false;
+	    	return;
+	    }
+
+	    // LOG_PRINT_L0("Output keys size: " << v.mv_size);
+	    if(v.mv_size != sizeof(output_data_t))
+	    {
+	    	txn.abort();
+	    	mdb_env_close(m_env);
+	    	m_open = false;
+	    	LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version.");
+	    	LOG_PRINT_RED_L0("Please delete the existing database and resync.");
+	    	return;
+	    }
+	}
+
   // commit the transaction
   txn.commit();
 
@@ -1464,19 +1637,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
   return *(const uint64_t*)result.mv_data;
 }
 
-//FIXME: make sure the random method used here is appropriate
-uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const
-{
-  LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  check_open();
-
-  uint64_t num_outputs = get_num_outputs(amount);
-  if (num_outputs == 0)
-    throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist"));
-
-  return crypto::rand<uint64_t>() % num_outputs;
-}
-
 uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@@ -1506,26 +1666,33 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
   return num_elems;
 }
 
-crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const
+output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
-  uint64_t glob_index = get_output_global_index(amount, index);
-
   mdb_txn_safe txn;
   if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
     throw0(DB_ERROR("Failed to create a transaction for the db"));
 
-  MDB_val_copy<uint64_t> k(glob_index);
+	MDB_val_copy<uint64_t> k(global_index);
   MDB_val v;
   auto get_result = mdb_get(txn, m_output_keys, &k, &v);
   if (get_result == MDB_NOTFOUND)
     throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist"));
   else if (get_result)
     throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
+	txn.commit();
+	return *(output_data_t *) v.mv_data;
+}
 
-  return *(crypto::public_key*)v.mv_data;
+output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index)
+{
+	LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+	check_open();
+
+	uint64_t glob_index = get_output_global_index(amount, index);
+	return get_output_key(glob_index);
 }
 
 tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const
@@ -1633,53 +1800,17 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
   return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data);
 }
 
-tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const
+tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
-  check_open();
+	std::vector < uint64_t > offsets;
-
+	std::vector<tx_out_index> indices;
-  mdb_txn_safe txn;
+	offsets.push_back(index);
-  mdb_txn_safe* txn_ptr = &txn;
+	get_output_tx_and_index(amount, offsets, indices);
-  if (m_batch_active)
+	if (!indices.size())
-    txn_ptr = m_write_txn;
-  else
-  {
-    if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
-      throw0(DB_ERROR("Failed to create a transaction for the db"));
-  }
-  lmdb_cur cur(*txn_ptr, m_output_amounts);
-
-  MDB_val_copy<uint64_t> k(amount);
-  MDB_val v;
-
-  auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
-  if (result == MDB_NOTFOUND)
     throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
-  else if (result)
-    throw0(DB_ERROR("DB error attempting to get an output"));
 
-  size_t num_elems = 0;
+	return indices[0];
-  mdb_cursor_count(cur, &num_elems);
-  if (num_elems <= index)
-    throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
-
-  mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
-
-  for (uint64_t i = 0; i < index; ++i)
-  {
-    mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
-  }
-
-  mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
-
-  uint64_t glob_index = *(const uint64_t*)v.mv_data;
-
-  cur.close();
-
-  if (! m_batch_active)
-    txn.commit();
-
-  return get_output_tx_and_index_from_global(glob_index);
 }
 
 std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const
@@ -1820,7 +1951,8 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
   return false;
 }
 
-void BlockchainLMDB::batch_start()
+// batch_num_blocks: (optional) Used to check if resize needed before batch transaction starts.
+void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   if (! m_batch_transactions)
@@ -1833,6 +1965,8 @@ void BlockchainLMDB::batch_start()
     throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
   check_open();
 
+  check_and_resize_for_batch(batch_num_blocks);
+
   m_write_batch_txn = new mdb_txn_safe();
 
   // NOTE: need to make sure it's destroyed properly when done
@@ -1915,19 +2049,16 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions)
   LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled"));
 }
 
-uint64_t BlockchainLMDB::add_block( const block& blk
+uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
-                                  , const size_t& block_size
+		const std::vector<transaction>& txs)
-                                  , const difficulty_type& cumulative_difficulty
-                                  , const uint64_t& coins_generated
-                                  , const std::vector<transaction>& txs
-                                  )
 {
   LOG_PRINT_L3("BlockchainLMDB::" << __func__);
   check_open();
 
   if (m_height % 1000 == 0)
   {
-    if (need_resize())
+    // for batch mode, DB resize check is done at start of batch transaction
+    if (! m_batch_active && need_resize())
     {
       LOG_PRINT_L0("LMDB memory map needs resized, doing that now.");
       do_resize();
@@ -2001,4 +2132,219 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
   --m_height;
 }
 
+void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
+		std::vector<tx_out_index> &tx_out_indices) const
+{
+	LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+	check_open();
+	tx_out_indices.clear();
+
+	mdb_txn_safe txn;
+	mdb_txn_safe* txn_ptr = &txn;
+	if (m_batch_active)
+		txn_ptr = m_write_txn;
+	else
+	{
+		if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+			throw0(DB_ERROR("Failed to create a transaction for the db"));
+	}
+
+	for (const uint64_t &index : global_indices)
+	{
+		MDB_val_copy<uint64_t> k(index);
+		MDB_val v;
+
+		auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v);
+		if (get_result == MDB_NOTFOUND)
+			throw1(OUTPUT_DNE("output with given index not in db"));
+		else if (get_result)
+			throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
+
+		crypto::hash tx_hash = *(crypto::hash*) v.mv_data;
+
+		get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v);
+		if (get_result == MDB_NOTFOUND)
+			throw1(OUTPUT_DNE("output with given index not in db"));
+		else if (get_result)
+			throw0(DB_ERROR("DB error attempting to fetch output tx index"));
+
+		auto result = tx_out_index(tx_hash, *(const uint64_t *) v.mv_data);
+		tx_out_indices.push_back(result);
+	}
+
+	if (!m_batch_active)
+		txn.commit();
+}
+
+void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets,
+		std::vector<uint64_t> &global_indices)
+{
+	LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+	TIME_MEASURE_START(txx);
+	check_open();
+	global_indices.clear();
+
+	uint64_t max = 0;
+	for (const uint64_t &index : offsets)
+	{
+		if (index > max)
+			max = index;
+	}
+
+	mdb_txn_safe txn;
+	mdb_txn_safe* txn_ptr = &txn;
+	if(m_batch_active)
+		txn_ptr = m_write_txn;
+	else
+	{
+		if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+			throw0(DB_ERROR("Failed to create a transaction for the db"));
+	}
+
+	lmdb_cur cur(*txn_ptr, m_output_amounts);
+
+	MDB_val_copy<uint64_t> k(amount);
+	MDB_val v;
+	auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
+	if (result == MDB_NOTFOUND)
+		throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
+	else if (result)
+		throw0(DB_ERROR("DB error attempting to get an output"));
+
+	size_t num_elems = 0;
+	mdb_cursor_count(cur, &num_elems);
+	if (max <= 1 && num_elems <= max)
+		throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
+
+	uint64_t t_dbmul = 0;
+	uint64_t t_dbscan = 0;
+	if (max <= 1)
+	{
+		for (const uint64_t& index : offsets)
+		{
+			mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
+			for (uint64_t i = 0; i < index; ++i)
+			{
+				mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
+			}
+
+			mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
+			uint64_t glob_index = *(const uint64_t*) v.mv_data;
+			LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index);
+			global_indices.push_back(glob_index);
+		}
+	}
+	else
+	{
+		uint32_t curcount = 0;
+		uint32_t blockstart = 0;
+		for (const uint64_t& index : offsets)
+		{
+			if (index >= num_elems)
+			{
+				LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index");
+				break;
+			}
+			while (index >= curcount)
+			{
+				TIME_MEASURE_START(db1);
+				if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0)
+				{
+					// allow partial results
+					result = false;
+					break;
+				}
+
+				int count = v.mv_size / sizeof(uint64_t);
+
+				blockstart = curcount;
+				curcount += count;
+				TIME_MEASURE_FINISH(db1);
+				t_dbmul += db1;
+			}
+
+			LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index);
+			TIME_MEASURE_START(db2);
+			uint64_t actual_index = index - blockstart;
+			uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index];
+
+			LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index);
+			global_indices.push_back(glob_index);
+
+			TIME_MEASURE_FINISH(db2);
+			t_dbscan += db2;
+
+		}
+	}
+
+	cur.close();
+	if(!m_batch_active)
+		txn.commit();
+
+	TIME_MEASURE_FINISH(txx);
+	LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan);
+}
+
+void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs)
+{
+	LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+	TIME_MEASURE_START(db3);
+	check_open();
+	outputs.clear();
+
+	std::vector <uint64_t> global_indices;
+	get_output_global_indices(amount, offsets, global_indices);
+
+	if (global_indices.size() > 0)
+	{
+    mdb_txn_safe txn;
+    mdb_txn_safe* txn_ptr = &txn;
+    if (m_batch_active)
+      txn_ptr = m_write_txn;
+    else
+    {
+      if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
+        throw0(DB_ERROR("Failed to create a transaction for the db"));
+    }
+		for (const uint64_t &index : global_indices)
+		{
+			MDB_val_copy<uint64_t> k(index);
+			MDB_val v;
+
+			auto get_result = mdb_get(*txn_ptr, m_output_keys, &k, &v);
+			if (get_result != 0)
+				throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist"));
+			else if (get_result)
+				throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
+
+			output_data_t data = *(output_data_t *) v.mv_data;
+			outputs.push_back(data);
+		}
+
+    if (!m_batch_active)
+      txn.commit();
+	}
+
+	TIME_MEASURE_FINISH(db3);
+	LOG_PRINT_L3("db3: " << db3);
+}
+
+void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices)
+{
+	LOG_PRINT_L3("BlockchainLMDB::" << __func__);
+	check_open();
+	indices.clear();
+
+	std::vector <uint64_t> global_indices;
+	get_output_global_indices(amount, offsets, global_indices);
+
+	TIME_MEASURE_START(db3);
+	if(global_indices.size() > 0)
+	{
+		get_output_tx_and_index_from_global(global_indices, indices);
+	}
+	TIME_MEASURE_FINISH(db3);
+	LOG_PRINT_L3("db3: " << db3);
+}
+
 }  // namespace cryptonote

src/blockchain_db/lmdb/db_lmdb.h

@@ -33,6 +33,8 @@
 
 #include <lmdb.h>
 
+#define ENABLE_AUTO_RESIZE
+
 namespace cryptonote
 {
 
@@ -155,11 +157,11 @@ public:
 
   virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
 
-  virtual uint64_t get_random_output(const uint64_t& amount) const;
-
   virtual uint64_t get_num_outputs(const uint64_t& amount) const;
 
-  virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const;
+  virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
+  virtual output_data_t get_output_key(const uint64_t& global_index) const;
+  virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs);
 
   virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const;
 
@@ -175,8 +177,12 @@ public:
   tx_out get_output(const uint64_t& index) const;
 
   virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
+  virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
+		  std::vector<tx_out_index> &tx_out_indices) const;
 
-  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
+  virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
+  virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
+  virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
 
   virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
   virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
@@ -191,17 +197,20 @@ public:
                             );
 
   virtual void set_batch_transactions(bool batch_transactions);
-  virtual void batch_start();
+  virtual void batch_start(uint64_t batch_num_blocks=0);
   virtual void batch_commit();
   virtual void batch_stop();
   virtual void batch_abort();
 
   virtual void pop_block(block& blk, std::vector<transaction>& txs);
 
+  virtual bool can_thread_bulk_indices() const { return true; }
 private:
-  void do_resize();
+  void do_resize(uint64_t size_increase=0);
 
-  bool need_resize() const;
+  bool need_resize(uint64_t threshold_size=0) const;
+  void check_and_resize_for_batch(uint64_t batch_num_blocks);
+  uint64_t get_estimated_batch_size(uint64_t batch_num_blocks) const;
 
   virtual void add_block( const block& blk
                 , const size_t& block_size
@@ -216,7 +225,7 @@ private:
 
   virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
 
-  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index);
+  virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
 
   virtual void remove_output(const tx_out& tx_output);
 
@@ -255,7 +264,7 @@ private:
    *
    * @return the global index of the desired output
    */
-  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const;
+  uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
 
   void check_open() const;
 
@@ -292,9 +301,18 @@ private:
   bool m_batch_transactions; // support for batch transactions
   bool m_batch_active; // whether batch transaction is in progress
 
-  constexpr static uint64_t DEFAULT_MAPSIZE = 1 << 30;
+#if defined(__arm__)
+  // force a value so it can compile with 32-bit ARM
+  constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 31;
+#else
+#if defined(ENABLE_AUTO_RESIZE)
+  constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 30;
+#else
+  constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 33;
+#endif
+#endif
+
   constexpr static float RESIZE_PERCENT = 0.8f;
-  constexpr static float RESIZE_FACTOR = 1.5f;
 };
 
 }  // namespace cryptonote

src/blockchain_utilities/README.md

@@ -4,13 +4,19 @@ Copyright (c) 2014-2015, The Monero Project
 
 ## Introduction
 
+The blockchain utilities allow one to convert an old style blockchain.bin file
+to a new style database. There are two ways to upgrade an old style blockchain:
+The recommended way is to run a `blockchain_export`, then `blockchain_import`.
+The other way is to run `blockchain_converter`. In both cases, you will be left
+with a new style blockchain.
+
 For importing into the LMDB database, compile with `DATABASE=lmdb`
 
 e.g.
 
 `DATABASE=lmdb make release`
 
-This is also the default compile setting on the blockchain branch.
+This is also the default compile setting on the master branch.
 
 By default, the exporter will use the original in-memory database (blockchain.bin) as its source.
 This default is to make migrating to an LMDB database easy, without having to recompile anything.
@@ -30,7 +36,8 @@ This loads the existing blockchain, for whichever database type it was compiled
 
 `$ blockchain_import`
 
-This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database.
+This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` (exported using the `blockchain_export` tool as described above)
+into the current database.
 
 Defaults: `--batch on`, `--batch size 20000`, `--verify on`
 
@@ -38,6 +45,9 @@ Batch size refers to number of blocks and can be adjusted for performance based
 
 Verification should only be turned off if importing from a trusted blockchain.
 
+If you encounter an error like "resizing not supported in batch mode", you can just re-run
+the `blockchain_import` command again, and it will restart from where it left off.
+
 ```bash
 ## use default settings to import blockchain.raw into database
 $ blockchain_import

src/blockchain_utilities/blockchain_converter.cpp

@@ -236,7 +236,7 @@ int main(int argc, char* argv[])
   }
 
   if (opt_batch)
-    blockchain->batch_start();
+    blockchain->batch_start(db_batch_size);
   uint64_t i = 0;
   for (i = start_block; i < end_block + 1; ++i)
   {
@@ -277,7 +277,7 @@ int main(int argc, char* argv[])
           std::cout << "\r                   \r";
           std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL;
           blockchain->batch_stop();
-          blockchain->batch_start();
+          blockchain->batch_start(db_batch_size);
           std::cout << ENDL;
           blockchain->show_stats();
         }

src/blockchain_utilities/blockchain_export.cpp

@@ -39,13 +39,14 @@ int main(int argc, char* argv[])
 {
   uint32_t log_level = 0;
   uint64_t block_stop = 0;
-  std::string import_filename = BLOCKCHAIN_RAW;
 
   boost::filesystem::path default_data_path {tools::get_default_data_dir()};
   boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
+  boost::filesystem::path output_file_path;
 
   po::options_description desc_cmd_only("Command line options");
   po::options_description desc_cmd_sett("Command line options and settings options");
+  const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
   const command_line::arg_descriptor<uint32_t> arg_log_level  = {"log-level",  "", log_level};
   const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
   const command_line::arg_descriptor<bool>     arg_testnet_on = {
@@ -57,6 +58,7 @@ int main(int argc, char* argv[])
 
   command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
   command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
+  command_line::add_arg(desc_cmd_sett, arg_output_file);
   command_line::add_arg(desc_cmd_sett, arg_testnet_on);
   command_line::add_arg(desc_cmd_sett, arg_log_level);
   command_line::add_arg(desc_cmd_sett, arg_block_stop);
@@ -97,9 +99,12 @@ int main(int argc, char* argv[])
 
   auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
   m_config_folder = command_line::get_arg(vm, data_dir_arg);
-  boost::filesystem::path output_dir {m_config_folder};
+
-  output_dir /= "export";
+  if (command_line::has_arg(vm, arg_output_file))
-  LOG_PRINT_L0("Export directory: " << output_dir.string());
+    output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file));
+  else
+    output_file_path = boost::filesystem::path(m_config_folder) / "export" / BLOCKCHAIN_RAW;
+  LOG_PRINT_L0("Export output file: " << output_file_path.string());
 
   // If we wanted to use the memory pool, we would set up a fake_core.
 
@@ -151,7 +156,7 @@ int main(int argc, char* argv[])
   LOG_PRINT_L0("Exporting blockchain raw data...");
 
   BootstrapFile bootstrap;
-  r = bootstrap.store_blockchain_raw(core_storage, NULL, output_dir, block_stop);
+  r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
   CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
   LOG_PRINT_L0("Blockchain raw data exported OK");
 }

src/blockchain_utilities/blockchain_import.cpp

@@ -173,7 +173,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
 }
 
 template <typename FakeCore>
-int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint64_t block_stop=0)
+int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
 {
 #if !defined(BLOCKCHAIN_DB)
   static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value,
@@ -188,11 +188,11 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
     simple_core.m_storage.get_db().reset_stats();
   }
 #endif
-  boost::filesystem::path raw_file_path(import_file_path);
+  boost::filesystem::path fs_import_file_path(import_file_path);
   boost::system::error_code ec;
-  if (!boost::filesystem::exists(raw_file_path, ec))
+  if (!boost::filesystem::exists(fs_import_file_path, ec))
   {
-    LOG_PRINT_L0("bootstrap file not found: " << raw_file_path);
+    LOG_PRINT_L0("bootstrap file not found: " << fs_import_file_path);
     return false;
   }
 
@@ -254,7 +254,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
   }
 
   if (use_batch)
-    simple_core.batch_start();
+    simple_core.batch_start(db_batch_size);
 
   LOG_PRINT_L0("Reading blockchain from bootstrap file...");
   std::cout << ENDL;
@@ -482,7 +482,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
             // zero-based height
             std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL;
             simple_core.batch_stop();
-            simple_core.batch_start();
+            simple_core.batch_start(db_batch_size);
             std::cout << ENDL;
 #if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
             simple_core.m_storage.get_db().show_stats();
@@ -526,7 +526,6 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
 
 int main(int argc, char* argv[])
 {
-  std::string import_filename = BLOCKCHAIN_RAW;
 #if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
   std::string default_db_engine = "memory";
 #else
@@ -536,14 +535,16 @@ int main(int argc, char* argv[])
   uint32_t log_level = LOG_LEVEL_0;
   uint64_t num_blocks = 0;
   uint64_t block_stop = 0;
-  std::string dirname;
+  std::string m_config_folder;
   std::string db_arg_str;
 
   boost::filesystem::path default_data_path {tools::get_default_data_dir()};
   boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
+  std::string import_file_path;
 
   po::options_description desc_cmd_only("Command line options");
   po::options_description desc_cmd_sett("Command line options and settings options");
+  const command_line::arg_descriptor<std::string> arg_input_file = {"input-file", "Specify input file", "", true};
   const command_line::arg_descriptor<uint32_t> arg_log_level   = {"log-level",  "", log_level};
   const command_line::arg_descriptor<uint64_t> arg_block_stop  = {"block-stop", "Stop at block number", block_stop};
   const command_line::arg_descriptor<uint64_t> arg_batch_size  = {"batch-size", "", db_batch_size};
@@ -571,6 +572,7 @@ int main(int argc, char* argv[])
 
   command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
   command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
+  command_line::add_arg(desc_cmd_sett, arg_input_file);
   command_line::add_arg(desc_cmd_sett, arg_testnet_on);
   command_line::add_arg(desc_cmd_sett, arg_log_level);
   command_line::add_arg(desc_cmd_sett, arg_database);
@@ -643,7 +645,7 @@ int main(int argc, char* argv[])
 
   opt_testnet = command_line::get_arg(vm, arg_testnet_on);
   auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
-  dirname = command_line::get_arg(vm, data_dir_arg);
+  m_config_folder = command_line::get_arg(vm, data_dir_arg);
   db_arg_str = command_line::get_arg(vm, arg_database);
 
   log_space::get_set_log_detalisation_level(true, log_level);
@@ -651,10 +653,15 @@ int main(int argc, char* argv[])
   LOG_PRINT_L0("Starting...");
   LOG_PRINT_L0("Setting log level = " << log_level);
 
-  boost::filesystem::path file_path {dirname};
+  boost::filesystem::path fs_import_file_path;
-  std::string import_file_path;
+
+  if (command_line::has_arg(vm, arg_input_file))
+    fs_import_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_input_file));
+  else
+    fs_import_file_path = boost::filesystem::path(m_config_folder) / "export" / BLOCKCHAIN_RAW;
+
+  import_file_path = fs_import_file_path.string();
 
-  import_file_path = (file_path / "export" / import_filename).string();
   if (command_line::has_arg(vm, arg_count_blocks))
   {
     BootstrapFile bootstrap;
@@ -694,7 +701,7 @@ int main(int argc, char* argv[])
   LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
 
   LOG_PRINT_L0("bootstrap file path: " << import_file_path);
-  LOG_PRINT_L0("database path:       " << file_path.string());
+  LOG_PRINT_L0("database path:       " << m_config_folder);
 
   try
   {
@@ -710,12 +717,12 @@ int main(int argc, char* argv[])
 #if !defined(BLOCKCHAIN_DB)
   if (db_engine == "lmdb")
   {
-    fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags);
+    fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
     import_from_file(simple_core, import_file_path, block_stop);
   }
   else if (db_engine == "memory")
   {
-    fake_core_memory simple_core(dirname, opt_testnet);
+    fake_core_memory simple_core(m_config_folder, opt_testnet);
     import_from_file(simple_core, import_file_path, block_stop);
   }
   else
@@ -732,9 +739,9 @@ int main(int argc, char* argv[])
     return 1;
   }
 #if BLOCKCHAIN_DB == DB_LMDB
-  fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags);
+  fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
 #else
-  fake_core_memory simple_core(dirname, opt_testnet);
+  fake_core_memory simple_core(m_config_folder, opt_testnet);
 #endif
 
   if (! vm["pop-blocks"].defaulted())

src/blockchain_utilities/bootstrap_file.cpp

@@ -50,8 +50,9 @@ namespace
 
 
 
-bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path)
+bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
 {
+  const boost::filesystem::path dir_path = file_path.parent_path();
   if (boost::filesystem::exists(dir_path))
   {
     if (!boost::filesystem::is_directory(dir_path))
@@ -69,7 +70,6 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path)
     }
   }
 
-  std::string file_path = (dir_path / BLOCKCHAIN_RAW).string();
   m_raw_data_file = new std::ofstream();
 
   bool do_initialize_file = false;
@@ -83,15 +83,15 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path)
   }
   else
   {
-    num_blocks = count_blocks(file_path);
+    num_blocks = count_blocks(file_path.string());
     LOG_PRINT_L0("appending to existing file with height: " << num_blocks-1 << "  total blocks: " << num_blocks);
   }
   m_height = num_blocks;
 
   if (do_initialize_file)
-    m_raw_data_file->open(file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);
+    m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
   else
-    m_raw_data_file->open(file_path, std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
+    m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
 
   if (m_raw_data_file->fail())
     return false;
@@ -286,9 +286,9 @@ bool BootstrapFile::close()
 
 
 #if SOURCE_DB == DB_MEMORY
-bool BootstrapFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_stop)
+bool BootstrapFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
 #else
-bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_stop)
+bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
 #endif
 {
   uint64_t num_blocks_written = 0;
@@ -297,7 +297,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
   m_tx_pool = _tx_pool;
   uint64_t progress_interval = 100;
   LOG_PRINT_L0("Storing blocks raw data...");
-  if (!BootstrapFile::open_writer(output_dir))
+  if (!BootstrapFile::open_writer(output_file))
   {
     LOG_PRINT_RED_L0("failed to open raw file for write");
     return false;
@@ -454,7 +454,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
     str1.assign(buf1, sizeof(chunk_size));
     if (! ::serialization::parse_binary(str1, chunk_size))
       throw std::runtime_error("Error in deserialization of chunk_size");
-    LOG_PRINT_L1("chunk_size: " << chunk_size);
+    LOG_PRINT_L3("chunk_size: " << chunk_size);
 
     if (chunk_size > BUFFER_SIZE)
     {