4rkal/monero
1
0
Fork 0
mirror of https://github.com/monero-project/monero.git synced 2025-01-28 18:56:31 +02:00
Code Issues Actions Packages Projects Releases Wiki Activity

Rebase

Browse Source
This commit is contained in:
Oran Juice 2015-08-29 22:47:47 +05:30
commit 3a51390716
170 changed files with 16437 additions and 7547 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
CMakeLists.txt
View File

@ -45,6 +45,40 @@ function (die msg)
message(FATAL_ERROR "${BoldRed}${msg}${ColourReset}")
endfunction ()
if (NOT ${ARCH} STREQUAL "")
string(SUBSTRING ${ARCH} 0 3 IS_ARM)
string(TOLOWER ${IS_ARM} IS_ARM)
if (${IS_ARM} STREQUAL "arm")
string(SUBSTRING ${ARCH} 0 5 ARM_TEST)
string(TOLOWER ${ARM_TEST} ARM_TEST)
if (${ARM_TEST} STREQUAL "armv6")
set(ARM6 1)
else()
set(ARM6 0)
endif()
if (${ARM_TEST} STREQUAL "armv7")
set(ARM7 1)
else()
set(ARM7 0)
endif()
endif()
endif()
if(WIN32 OR ARM7 OR ARM6)
set(CMAKE_C_FLAGS_RELEASE "-O2 -DNDEBUG")
set(CMAKE_CXX_FLAGS_RELEASE "-O2 -DNDEBUG")
endif()
# set this to 0 if per-block checkpoint needs to be disabled
set(PER_BLOCK_CHECKPOINT 1)
if(PER_BLOCK_CHECKPOINT)
add_definitions("-DPER_BLOCK_CHECKPOINT")
endif()
list(INSERT CMAKE_MODULE_PATH 0
"${CMAKE_SOURCE_DIR}/cmake")
@ -127,16 +161,16 @@ endif()
option(STATIC "Link libraries statically" ${DEFAULT_STATIC})
if(MINGW)
get_filename_component(msys2_install_path "[HKEY_CURRENT_USER\\Software\\Microsoft\\Windows\\CurrentVersion\\Uninstall\\MSYS2 ${ARCH_WIDTH}bit;InstallLocation]" ABSOLUTE)
string(REGEX MATCH "^[^/]:/[^/]*" msys2_install_path "${CMAKE_C_COMPILER}")
message(STATUS "MSYS location: ${msys2_install_path}")
set(CMAKE_INCLUDE_PATH "${msys2_install_path}/mingw${ARCH_WIDTH}/include")
# This is necessary because otherwise CMake will make Boost libraries -lfoo
# rather than a full path. Unfortunately, this makes the shared libraries get
# linked due to a bug in CMake which misses putting -static flags around the
# -lfoo arguments.
list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES
"${msys2_install_path}/mingw${ARCH_WIDTH}/lib")
list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES
"${msys2_install_path}/mingw${ARCH_WIDTH}/lib")
set(DEFLIB ${msys2_install_path}/mingw${ARCH_WIDTH}/lib)
list(REMOVE_ITEM CMAKE_C_IMPLICIT_LINK_DIRECTORIES ${DEFLIB})
list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_DIRECTORIES ${DEFLIB})
endif()
if(STATIC)
@ -158,18 +192,47 @@ if (DEFINED ENV{DATABASE})
else()
message(STATUS "Could not find DATABASE in env (not required unless you want to change database type from default: ${DATABASE})")
endif()
set(BERKELEY_DB 0)
if (DATABASE STREQUAL "lmdb")
set(BLOCKCHAIN_DB DB_LMDB)
# temporarily allow mingw to compile with berkeley_db,
# regardless if building static or not
if(NOT STATIC OR MINGW)
find_package(BerkeleyDB)
if(NOT BERKELEY_DB_LIBRARIES)
message(STATUS "BerkeleyDB not found and has been disabled.")
else()
message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
if(BERKELEY_DB_LIBRARIES)
message(STATUS "Found BerkeleyDB shared library")
set(BDB_STATIC false CACHE BOOL "BDB Static flag")
set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
set(BERKELEY_DB 1)
else()
message(STATUS "Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
endif()
endif()
endif()
elseif (DATABASE STREQUAL "memory")
set(BLOCKCHAIN_DB DB_MEMORY)
else()
die("Invalid database type: ${DATABASE}")
endif()
if(BERKELEY_DB)
add_definitions("-DBERKELEY_DB")
endif()
add_definitions("-DBLOCKCHAIN_DB=${BLOCKCHAIN_DB}")
if (UNIX AND NOT APPLE)
# Note that at the time of this writing the -Wstrict-prototypes flag added below will make this fail
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads)
endif()
@ -194,7 +257,7 @@ include_directories(external/rapidjson)
include_directories(${LMDB_INCLUDE})
# Final setup for Berkeley DB
if (NOT STATIC)
if (BERKELEY_DB)
include_directories(${BDB_INCLUDE})
endif()
@ -210,10 +273,15 @@ if(MSVC)
include_directories(SYSTEM src/platform/msc)
else()
set(ARCH native CACHE STRING "CPU to build for: -march value or default")
if(ARCH STREQUAL "default")
# -march=armv7-a conflicts with -mcpu=cortex-a7
if(ARCH STREQUAL "default" OR ARM7)
set(ARCH_FLAG "")
else()
set(ARCH_FLAG "-march=${ARCH}")
if(ARCH STREQUAL "x86_64")
set(ARCH_FLAG "-march=x86-64")
else()
set(ARCH_FLAG "-march=${ARCH}")
endif()
endif()
set(WARNINGS "-Wall -Wextra -Wpointer-arith -Wundef -Wvla -Wwrite-strings -Wno-error=extra -Wno-error=deprecated-declarations -Wno-error=sign-compare -Wno-error=strict-aliasing -Wno-error=type-limits -Wno-unused-parameter -Wno-error=unused-variable -Wno-error=undef -Wno-error=uninitialized")
if(CMAKE_C_COMPILER_ID STREQUAL "Clang")
@ -256,14 +324,18 @@ else()
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -D_GNU_SOURCE ${MINGW_FLAG} ${WARNINGS} ${CXX_WARNINGS} ${ARCH_FLAG} -maes")
endif()
string(SUBSTRING ${ARCH} 0 3 ARM_TEST)
string(TOLOWER ${ARM_TEST} ARM_TEST)
if(${ARM_TEST} STREQUAL "arm")
message(STATUS "Setting ARM C and C++ flags")
if(ARM6)
message(STATUS "Setting ARM6 C and C++ flags")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfpu=vfp -mfloat-abi=hard")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mfpu=vfp -mfloat-abi=hard")
endif()
if(ARM7)
message(STATUS "Setting ARM7 C and C++ flags")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -O2 -mcpu=cortex-a7 -mfloat-abi=hard -mfpu=vfpv4 -funsafe-math-optimizations -mtune=cortex-a7")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -mcpu=cortex-a7 -mfloat-abi=hard -mfpu=vfpv4 -funsafe-math-optimizations -mtune=cortex-a7")
endif()
if(APPLE)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DGTEST_HAS_TR1_TUPLE=0")
endif()
@ -341,7 +413,7 @@ elseif(APPLE OR FREEBSD)
set(EXTRA_LIBRARIES "")
elseif(NOT MSVC)
find_library(RT rt)
set(EXTRA_LIBRARIES ${RT} ${PTHREAD} ${DL})
set(EXTRA_LIBRARIES ${RT} ${DL})
endif()
include(version.cmake)

12
Makefile
View File

@ -58,6 +58,14 @@ release-all:
mkdir -p build/release
cd build/release && cmake -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
release-arm6:
mkdir -p build/release
cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv6zk" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
release-arm7:
mkdir -p build/release
cd build/release && cmake -D BUILD_TESTS=OFF -D ARCH="armv7-a" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
release-static: release-static-64
release-static-64:
@ -68,10 +76,6 @@ release-static-32:
mkdir -p build/release
cd build/release && cmake -D STATIC=ON -D ARCH="i686" -D BUILD_64=OFF -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
release-static-arm6:
mkdir -p build/release
cd build/release && cmake -D STATIC=ON -D ARCH="armv6zk" -D BUILD_64=OFF -D NO_AES=ON -D CMAKE_BUILD_TYPE=release ../.. && $(MAKE)
release-static-win64:
mkdir -p build/release
cd build/release && cmake -G "MSYS Makefiles" -D STATIC=ON -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_BUILD_TYPE=Release -D CMAKE_TOOLCHAIN_FILE=../../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ../.. && $(MAKE)

45
README.i18n Normal file
View File

@ -0,0 +1,45 @@
The Monero command line tools can be translated in various languages.
In order to use the same translation workflow as the future GUI, they
use Qt Linguist translation files. However, to avoid the dependencies
on Qt this normally implies, they use a custom loader to read those
files at runtime. In order to update, or build translations files, you
do need to have Qt tools installed, however. For translating, you need
either the Qt Linguist GUI, or another tool that supports Qt ts files,
such as Transifex. To run, you do not need anything Qt.
To update ts files after changing source code:
./utils/translations/update-translations.sh
To add a new language, eg Spanish (ISO code es):
cp transations/monero.ts transations/monero_es.ts
To edit translations for Spanish:
linguist translations/monero_es.ts
To build translations after modiying them:
./utils/translations/build-translations.sh
To test a translation:
LANG=es ./build/release/bin/simplewallet
To add new translatable sources in the source:
Use the tr(string) function if possible. If the code is in a class,
and this class doesn't already have a tr() static function, add one,
which uses a context named after what lupdate uses for the context,
usually the fully qualified class name (eg, cryptonote::simple_wallet).
If you need to use tr in code that's not in a class, you can use the
fully qualified version (eg, simple_wallet::tr) of the one matching
the context you want.
Use QT_TRANSLATE_NOOP(string) if you want to specify a context manually.
If you're getting messages of the form:
Class 'cryptonote::simple_wallet' lacks Q_OBJECT macro
all is fine, we don't actually need that here.

7
README.md
View File

@ -134,7 +134,7 @@ cd build
```
* If you are on a 64-bit system, run:
```
cmake -G "MSYS Makefiles" -D CMAKE_BUILD_TYPE=Release -D ARCH="x86_64" -D BUILD_64=ON -D CMAKE_TOOLCHAIN_FILE=../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ..
cmake -G "MSYS Makefiles" -D CMAKE_BUILD_TYPE=Release -D ARCH="x86-64" -D BUILD_64=ON -D CMAKE_TOOLCHAIN_FILE=../cmake/64-bit-toolchain.cmake -D MSYS2_FOLDER=c:/msys64 ..
```
* If you are on a 32-bit system, run:
```
@ -167,3 +167,8 @@ Dependencies: Doxygen 1.8.0 or later, Graphviz 2.28 or later (optional).
* To build, change to the root of the source code directory, and run `doxygen Doxyfile`
* If you have installed Graphviz, you can also generate in-doc diagrams by instead running `HAVE_DOT=YES doxygen Doxyfile`
* The output will be built in doc/html/
## Internationalization
See README.i18n

12
contrib/epee/include/console_handler.h
View File

@ -274,14 +274,15 @@ namespace epee
}
std::string command;
if(!m_stdin_reader.get_line(command))
{
LOG_PRINT("Failed to read line.", LOG_LEVEL_0);
}
bool get_line_ret = m_stdin_reader.get_line(command);
if (m_stdin_reader.eos())
{
break;
}
if (!get_line_ret)
{
LOG_PRINT("Failed to read line.", LOG_LEVEL_0);
}
string_tools::trim(command);
LOG_PRINT_L2("Read command: " << command);
@ -303,7 +304,8 @@ namespace epee
std::cout << usage;
}
}
exit_handler();
if (exit_handler)
exit_handler();
return true;
CATCH_ENTRY_L0("console_handler", false);
}

3
contrib/epee/include/misc_log_ex.h
View File

@ -861,7 +861,8 @@ namespace log_space
std::string::size_type a = m_default_log_file.rfind('.');
if ( a != std::string::npos )
m_default_log_file.erase( a, m_default_log_file.size());
m_default_log_file += ".log";
if ( ! m_default_log_file.empty() )
m_default_log_file += ".log";
return true;
}

20
external/db_drivers/CMakeLists.txt vendored
View File

@ -31,24 +31,4 @@
message(STATUS "Using ${ARCH_WIDTH}-bit LMDB from source tree")
add_subdirectory(liblmdb${ARCH_WIDTH})
set(LMDB_INCLUDE "${CMAKE_CURRENT_SOURCE_DIR}/liblmdb${ARCH_WIDTH}" CACHE STRING "LMDB Include path")
set(LMDB_LIBRARY "lmdb" CACHE STRING "LMDB Library name")
if (NOT STATIC)
find_package(BerkeleyDB)
if(NOT BERKELEY_DB_LIBRARIES)
die("BerkeleyDB not found. At this time it should be installed in your system for a non-static build.")
else()
message(STATUS "Found BerkeleyDB include (db.h) in ${BERKELEY_DB_INCLUDE_DIR}")
if(BERKELEY_DB_LIBRARIES)
message(STATUS "Found BerkeleyDB shared library")
set(BDB_STATIC false CACHE BOOL "BDB Static flag")
set(BDB_INCLUDE ${BERKELEY_DB_INCLUDE_DIR} CACHE STRING "BDB include path")
set(BDB_LIBRARY ${BERKELEY_DB_LIBRARIES} CACHE STRING "BDB library name")
set(BDB_LIBRARY_DIRS "" CACHE STRING "BDB Library dirs")
else()
die("Found BerkeleyDB includes, but could not find BerkeleyDB library. Please make sure you have installed libdb and libdb-dev or the equivalent")
endif()
endif()
endif()

28
external/db_drivers/liblmdb32/CHANGES vendored
View File

@ -1,5 +1,33 @@
LMDB 0.9 Change Log
LMDB 0.9.15 Release (2015/06/19)
Fix txn init (ITS#7961,#7987)
Fix MDB_PREV_DUP (ITS#7955,#7671)
Fix compact of empty env (ITS#7956)
Fix mdb_copy file mode
Fix mdb_env_close() after failed mdb_env_open()
Fix mdb_rebalance collapsing root (ITS#8062)
Fix mdb_load with large values (ITS#8066)
Fix to retry writes on EINTR (ITS#8106)
Fix mdb_cursor_del on empty DB (ITS#8109)
Fix MDB_INTEGERDUP key compare (ITS#8117)
Fix error handling (ITS#7959,#8157,etc.)
Fix race conditions (ITS#7969,7970)
Added workaround for fdatasync bug in ext3fs
Build
Don't use -fPIC for static lib
Update .gitignore (ITS#7952,#7953)
Cleanup for "make test" (ITS#7841), "make clean", mtest*.c
Misc. Android/Windows cleanup
Documentation
Fix MDB_APPEND doc
Fix MDB_MAXKEYSIZE doc (ITS#8156)
Fix mdb_cursor_put,mdb_cursor_del EACCES description
Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021)
Clarify MDB_WRITEMAP doc (ITS#8021)
Clarify mdb_env_open doc
Clarify mdb_dbi_open doc
LMDB 0.9.14 Release (2014/09/20)
Fix to support 64K page size (ITS#7713)
Fix to persist decreased as well as increased mapsizes (ITS#7789)

7
external/db_drivers/liblmdb32/CMakeLists.txt vendored
View File

@ -26,6 +26,13 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
if(FREEBSD)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMDB_DSYNC=O_SYNC")
endif()
# pass the VL32 flag so that we actually invoke the VL32 extensions
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DVL32")
set (lmdb_sources
mdb.c
midl.c)

2
external/db_drivers/liblmdb32/COPYRIGHT vendored
View File

@ -1,4 +1,4 @@
Copyright 2011-2014 Howard Chu, Symas Corp.
Copyright 2011-2015 Howard Chu, Symas Corp.
All rights reserved.
Redistribution and use in source and binary forms, with or without

19
external/db_drivers/liblmdb32/Makefile vendored
View File

@ -11,6 +11,7 @@
# - MDB_USE_POSIX_SEM
# - MDB_DSYNC
# - MDB_FDATASYNC
# - MDB_FDATASYNC_WORKS
# - MDB_USE_PWRITEV
#
# There may be other macros in mdb.c of interest. You should
@ -42,18 +43,18 @@ install: $(ILIBS) $(IPROGS) $(IHDRS)
for f in $(IDOCS); do cp $$f $(DESTDIR)$(prefix)/man/man1; done
clean:
rm -rf $(PROGS) *.[ao] *.so *~ testdb
rm -rf $(PROGS) *.[ao] *.[ls]o *~ testdb
test: all
mkdir testdb
rm -rf testdb && mkdir testdb
./mtest && ./mdb_stat testdb
liblmdb.a: mdb.o midl.o
ar rs $@ mdb.o midl.o
liblmdb.so: mdb.o midl.o
liblmdb.so: mdb.lo midl.lo
# $(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.o midl.o $(SOLIBS)
$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
mdb_stat: mdb_stat.o liblmdb.a
mdb_copy: mdb_copy.o liblmdb.a
@ -67,10 +68,16 @@ mtest5: mtest5.o liblmdb.a
mtest6: mtest6.o liblmdb.a
mdb.o: mdb.c lmdb.h midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c
$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c
midl.o: midl.c midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c
$(CC) $(CFLAGS) $(CPPFLAGS) -c midl.c
mdb.lo: mdb.c lmdb.h midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c mdb.c -o $@
midl.lo: midl.c midl.h
$(CC) $(CFLAGS) -fPIC $(CPPFLAGS) -c midl.c -o $@
%: %.o
$(CC) $(CFLAGS) $(LDFLAGS) $^ $(LDLIBS) -o $@

57
external/db_drivers/liblmdb32/lmdb.h vendored
View File

@ -119,7 +119,7 @@
*
* @author Howard Chu, Symas Corporation.
*
* @copyright Copyright 2011-2014 Howard Chu, Symas Corp. All rights reserved.
* @copyright Copyright 2011-2015 Howard Chu, Symas Corp. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted only as authorized by the OpenLDAP
@ -184,7 +184,7 @@ typedef int mdb_filehandle_t;
/** Library minor version */
#define MDB_VERSION_MINOR 9
/** Library patch version */
#define MDB_VERSION_PATCH 14
#define MDB_VERSION_PATCH 15
/** Combine args a,b,c into a single integer for easy version comparisons */
#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c))
@ -194,7 +194,7 @@ typedef int mdb_filehandle_t;
MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
/** The release date of this library version */
#define MDB_VERSION_DATE "September 20, 2014"
#define MDB_VERSION_DATE "June 19, 2015"
/** A stringifier for the version info */
#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"
@ -296,12 +296,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
#define MDB_REVERSEKEY 0x02
/** use sorted duplicates */
#define MDB_DUPSORT 0x04
/** numeric keys in native byte order.
/** numeric keys in native byte order: either unsigned int or size_t.
* The keys must all be of the same size. */
#define MDB_INTEGERKEY 0x08
/** with #MDB_DUPSORT, sorted dup items have fixed size */
#define MDB_DUPFIXED 0x10
/** with #MDB_DUPSORT, dups are numeric in native byte order */
/** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
#define MDB_INTEGERDUP 0x20
/** with #MDB_DUPSORT, use reverse string dups */
#define MDB_REVERSEDUP 0x40
@ -516,8 +516,8 @@ int mdb_env_create(MDB_env **env);
* and uses fewer mallocs, but loses protection from application bugs
* like wild pointer writes and other bad updates into the database.
* Incompatible with nested transactions.
* Processes with and without MDB_WRITEMAP on the same environment do
* not cooperate well.
* Do not mix processes with and without MDB_WRITEMAP on the same
* environment. This can defeat durability (#mdb_env_sync etc).
* <li>#MDB_NOMETASYNC
* Flush system buffers to disk only once per transaction, omit the
* metadata flush. Defer that until the system flushes files to disk,
@ -588,8 +588,8 @@ int mdb_env_create(MDB_env **env);
* reserved in that case.
* This flag may be changed at any time using #mdb_env_set_flags().
* </ul>
* @param[in] mode The UNIX permissions to set on created files. This parameter
* is ignored on Windows.
* @param[in] mode The UNIX permissions to set on created files and semaphores.
* This parameter is ignored on Windows.
* @return A non-zero error value on failure and 0 on success. Some possible
* errors are:
* <ul>
@ -698,7 +698,8 @@ int mdb_env_info(MDB_env *env, MDB_envinfo *stat);
* Data is always written to disk when #mdb_txn_commit() is called,
* but the operating system may keep it buffered. LMDB always flushes
* the OS buffers upon commit as well, unless the environment was
* opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC.
* opened with #MDB_NOSYNC or in part #MDB_NOMETASYNC. This call is
* not valid if the environment was opened with #MDB_RDONLY.
* @param[in] env An environment handle returned by #mdb_env_create()
* @param[in] force If non-zero, force a synchronous flush. Otherwise
* if the environment has the #MDB_NOSYNC flag set the flushes
@ -706,6 +707,7 @@ int mdb_env_info(MDB_env *env, MDB_envinfo *stat);
* @return A non-zero error value on failure and 0 on success. Some possible
* errors are:
* <ul>
* <li>EACCES - the environment is read-only.
* <li>EINVAL - an invalid parameter was specified.
* <li>EIO - an error occurred during synchronization.
* </ul>
@ -1019,15 +1021,17 @@ int mdb_txn_renew(MDB_txn *txn);
* The database handle may be discarded by calling #mdb_dbi_close().
* The old database handle is returned if the database was already open.
* The handle may only be closed once.
*
* The database handle will be private to the current transaction until
* the transaction is successfully committed. If the transaction is
* aborted the handle will be closed automatically.
* After a successful commit the
* handle will reside in the shared environment, and may be used
* by other transactions. This function must not be called from
* multiple concurrent transactions. A transaction that uses this function
* must finish (either commit or abort) before any other transaction may
* use this function.
* After a successful commit the handle will reside in the shared
* environment, and may be used by other transactions.
*
* This function must not be called from multiple concurrent
* transactions in the same process. A transaction that uses
* this function must finish (either commit or abort) before
* any other transaction in the process may use this function.
*
* To use named databases (with name != NULL), #mdb_env_set_maxdbs()
* must be called before opening the environment. Database names
@ -1048,9 +1052,9 @@ int mdb_txn_renew(MDB_txn *txn);
* keys may have multiple data items, stored in sorted order.) By default
* keys must be unique and may have only a single data item.
* <li>#MDB_INTEGERKEY
* Keys are binary integers in native byte order. Setting this option
* requires all keys to be the same size, typically sizeof(int)
* or sizeof(size_t).
* Keys are binary integers in native byte order, either unsigned int
* or size_t, and will be sorted as such.
* The keys must all be of the same size.
* <li>#MDB_DUPFIXED
* This flag may only be used in combination with #MDB_DUPSORT. This option
* tells the library that the data items for this database are all the same
@ -1058,8 +1062,8 @@ int mdb_txn_renew(MDB_txn *txn);
* all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
* cursor operations may be used to retrieve multiple items at once.
* <li>#MDB_INTEGERDUP
* This option specifies that duplicate data items are also integers, and
* should be sorted as such.
* This option specifies that duplicate data items are binary integers,
* similar to #MDB_INTEGERKEY keys.
* <li>#MDB_REVERSEDUP
* This option specifies that duplicate data items should be compared as
* strings in reverse order.
@ -1270,10 +1274,9 @@ int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
* LMDB does nothing else with this memory, the caller is expected
* to modify all of the space requested.
* <li>#MDB_APPEND - append the given key/data pair to the end of the
* database. No key comparisons are performed. This option allows
* fast bulk loading when keys are already known to be in the
* correct order. Loading unsorted keys with this flag will cause
* data corruption.
* database. This option allows fast bulk loading when keys are
* already known to be in the correct order. Loading unsorted keys
* with this flag will cause a #MDB_KEYEXIST error.
* <li>#MDB_APPENDDUP - as above, but for sorted dup data.
* </ul>
* @return A non-zero error value on failure and 0 on success. Some possible
@ -1449,7 +1452,7 @@ int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
* <ul>
* <li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
* <li>#MDB_TXN_FULL - the transaction has too many dirty pages.
* <li>EACCES - an attempt was made to modify a read-only database.
* <li>EACCES - an attempt was made to write in a read-only transaction.
* <li>EINVAL - an invalid parameter was specified.
* </ul>
*/
@ -1469,7 +1472,7 @@ int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
* @return A non-zero error value on failure and 0 on success. Some possible
* errors are:
* <ul>
* <li>EACCES - an attempt was made to modify a read-only database.
* <li>EACCES - an attempt was made to write in a read-only transaction.
* <li>EINVAL - an invalid parameter was specified.
* </ul>
*/

328
external/db_drivers/liblmdb32/mdb.c vendored
View File

@ -5,7 +5,7 @@
* BerkeleyDB API, but much simplified.
*/
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -79,6 +79,14 @@ extern int cacheflush(char *addr, int nbytes, int cache);
#define CACHEFLUSH(addr, bytes, cache)
#endif
#if defined(__linux) && !defined(MDB_FDATASYNC_WORKS)
/** fdatasync is broken on ext3/ext4fs on older kernels, see
* description in #mdb_env_open2 comments. You can safely
* define MDB_FDATASYNC_WORKS if this code will only be run
* on kernels 3.6 and newer.
*/
#define BROKEN_FDATASYNC
#endif
#include <errno.h>
#include <limits.h>
@ -438,12 +446,17 @@ static txnid_t mdb_debug_start;
/** The version number for a database's lockfile format. */
#define MDB_LOCK_VERSION 1
/** @brief The max size of a key we can write, or 0 for dynamic max.
/** @brief The max size of a key we can write, or 0 for computed max.
*
* Define this as 0 to compute the max from the page size. 511
* is default for backwards compat: liblmdb <= 0.9.10 can break
* when modifying a DB with keys/dupsort data bigger than its max.
* #MDB_DEVEL sets the default to 0.
* This macro should normally be left alone or set to 0.
* Note that a database with big keys or dupsort data cannot be
* reliably modified by a liblmdb which uses a smaller max.
* The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
*
* Other values are allowed, for backwards compat. However:
* A value bigger than the computed max can break if you do not
* know what you are doing, and liblmdb <= 0.9.10 can break when
* modifying a DB with keys/dupsort data bigger than its max.
*
* Data items in an #MDB_DUPSORT database are also limited to
* this size, since they're actually keys of a sub-DB. Keys and
@ -580,11 +593,11 @@ typedef struct MDB_rxbody {
* started from so we can avoid overwriting any data used in that
* particular version.
*/
txnid_t mrb_txnid;
volatile txnid_t mrb_txnid;
/** The process ID of the process owning this reader txn. */
MDB_PID_T mrb_pid;
volatile MDB_PID_T mrb_pid;
/** The thread ID of the thread owning this txn. */
MDB_THR_T mrb_tid;
volatile MDB_THR_T mrb_tid;
} MDB_rxbody;
/** The actual reader record, with cacheline padding. */
@ -632,12 +645,12 @@ typedef struct MDB_txbody {
* This is recorded here only for convenience; the value can always
* be determined by reading the main database meta pages.
*/
txnid_t mtb_txnid;
volatile txnid_t mtb_txnid;
/** The number of slots that have been used in the reader table.
* This always records the maximum count, it is not decremented
* when readers release their slots.
*/
unsigned mtb_numreaders;
volatile unsigned mtb_numreaders;
} MDB_txbody;
/** The actual reader table definition. */
@ -898,7 +911,7 @@ typedef struct MDB_meta {
/** Stamp identifying this as an LMDB file. It must be set
* to #MDB_MAGIC. */
uint32_t mm_magic;
/** Version number of this lock file. Must be set to #MDB_DATA_VERSION. */
/** Version number of this file. Must be set to #MDB_DATA_VERSION. */
uint32_t mm_version;
void *mm_address; /**< address for fixed mapping */
size_t mm_mapsize; /**< size of mmap region */
@ -908,7 +921,7 @@ typedef struct MDB_meta {
/** Any persistent environment flags. @ref mdb_env */
#define mm_flags mm_dbs[0].md_flags
pgno_t mm_last_pg; /**< last used page in file */
txnid_t mm_txnid; /**< txnid that committed this page */
volatile txnid_t mm_txnid; /**< txnid that committed this page */
} MDB_meta;
/** Buffer for a stack-allocated meta page.
@ -991,7 +1004,7 @@ struct MDB_txn {
#ifdef VL32
/** List of read-only pages */
MDB_ID2L mt_rpages;
#endif
#endif
/** Number of DB records in use. This number only ever increments;
* we don't decrement it when individual DB handles are closed.
*/
@ -1103,6 +1116,8 @@ struct MDB_env {
#define MDB_ENV_ACTIVE 0x20000000U
/** me_txkey is set */
#define MDB_ENV_TXKEY 0x10000000U
/** fdatasync is unreliable */
#define MDB_FSYNCONLY 0x08000000U
uint32_t me_flags; /**< @ref mdb_env */
unsigned int me_psize; /**< DB page size, inited from me_os_psize */
unsigned int me_os_psize; /**< OS page size, from #GET_PAGESIZE */
@ -1242,6 +1257,13 @@ static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
/** @endcond */
/** Compare two items pointing at size_t's of unknown alignment. */
#ifdef MISALIGNED_OK
# define mdb_cmp_clong mdb_cmp_long
#else
# define mdb_cmp_clong mdb_cmp_cint
#endif
#ifdef _WIN32
static SECURITY_DESCRIPTOR mdb_null_sd;
static SECURITY_ATTRIBUTES mdb_all_sa;
@ -1323,7 +1345,7 @@ mdb_strerror(int err)
buf[0] = 0;
FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM |
FORMAT_MESSAGE_IGNORE_INSERTS,
NULL, err, 0, ptr, sizeof(buf), pad);
NULL, err, 0, ptr, sizeof(buf), (va_list *)pad);
return ptr;
#else
return strerror(err);
@ -1555,7 +1577,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
int
mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
{
return txn->mt_dbxs[dbi].md_dcmp(a, b);
MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
#if UINT_MAX < SIZE_MAX
if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
dcmp = mdb_cmp_clong;
#endif
return dcmp(a, b);
}
/** Allocate memory for a page.
@ -2323,6 +2350,8 @@ int
mdb_env_sync(MDB_env *env, int force)
{
int rc = 0;
if (env->me_flags & MDB_RDONLY)
return EACCES;
if (force || !F_ISSET(env->me_flags, MDB_NOSYNC)) {
if (env->me_flags & MDB_WRITEMAP) {
int flags = ((env->me_flags & MDB_MAPASYNC) && !force)
@ -2334,6 +2363,12 @@ mdb_env_sync(MDB_env *env, int force)
rc = ErrCode();
#endif
} else {
#ifdef BROKEN_FDATASYNC
if (env->me_flags & MDB_FSYNCONLY) {
if (fsync(env->me_fd))
rc = ErrCode();
} else
#endif
if (MDB_FDATASYNC(env->me_fd))
rc = ErrCode();
}
@ -2489,15 +2524,11 @@ mdb_txn_renew0(MDB_txn *txn)
MDB_env *env = txn->mt_env;
MDB_txninfo *ti = env->me_txns;
MDB_meta *meta;
unsigned int i, nr;
unsigned int i, nr, flags = txn->mt_flags;
uint16_t x;
int rc, new_notls = 0;
/* Setup db info */
txn->mt_numdbs = env->me_numdbs;
txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */
if (txn->mt_flags & MDB_TXN_RDONLY) {
if ((flags &= MDB_TXN_RDONLY) != 0) {
if (!ti) {
meta = env->me_metas[ mdb_env_pick_meta(env) ];
txn->mt_txnid = meta->mm_txnid;
@ -2543,10 +2574,14 @@ mdb_txn_renew0(MDB_txn *txn)
return rc;
}
}
txn->mt_txnid = r->mr_txnid = ti->mti_txnid;
do /* LY: Retry on a race, ITS#7970. */
r->mr_txnid = ti->mti_txnid;
while(r->mr_txnid != ti->mti_txnid);
txn->mt_txnid = r->mr_txnid;
txn->mt_u.reader = r;
meta = env->me_metas[txn->mt_txnid & 1];
}
txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */
} else {
if (ti) {
LOCK_MUTEX_W(env);
@ -2562,6 +2597,9 @@ mdb_txn_renew0(MDB_txn *txn)
if (txn->mt_txnid == mdb_debug_start)
mdb_debug = 1;
#endif
txn->mt_child = NULL;
txn->mt_loose_pgs = NULL;
txn->mt_loose_count = 0;
txn->mt_dirty_room = MDB_IDL_UM_MAX;
txn->mt_u.dirty_list = env->me_dirty_list;
txn->mt_u.dirty_list[0].mid = 0;
@ -2578,6 +2616,10 @@ mdb_txn_renew0(MDB_txn *txn)
/* Moved to here to avoid a data race in read TXNs */
txn->mt_next_pgno = meta->mm_last_pg+1;
txn->mt_flags = flags;
/* Setup db info */
txn->mt_numdbs = env->me_numdbs;
for (i=2; i<txn->mt_numdbs; i++) {
x = env->me_dbflags[i];
txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
@ -2643,18 +2685,16 @@ mdb_txn_begin(MDB_env *env, MDB_txn *parent, unsigned int flags, MDB_txn **ret)
}
tsize = sizeof(MDB_ntxn);
}
size = tsize + env->me_maxdbs * (sizeof(MDB_db)+1);
size = tsize;
if (!(flags & MDB_RDONLY)) {
if (!parent) {
txn = env->me_txn0;
txn->mt_flags = 0;
txn = env->me_txn0; /* just reuse preallocated write txn */
goto ok;
}
/* child txns use own copy of cursors */
size += env->me_maxdbs * sizeof(MDB_cursor *);
/* child txns use parent's dbiseqs */
if (!parent)
size += env->me_maxdbs * sizeof(unsigned int);
}
size += env->me_maxdbs * (sizeof(MDB_db)+1);
if ((txn = calloc(1, size)) == NULL) {
DPRINTF(("calloc: %s", strerror(errno)));
@ -2807,38 +2847,40 @@ mdb_txn_reset0(MDB_txn *txn, const char *act)
txn->mt_numdbs = 0; /* close nothing if called again */
txn->mt_dbxs = NULL; /* mark txn as reset */
} else {
mdb_cursors_close(txn, 0);
pgno_t *pghead = env->me_pghead;
mdb_cursors_close(txn, 0);
if (!(env->me_flags & MDB_WRITEMAP)) {
mdb_dlist_free(txn);
}
mdb_midl_free(env->me_pghead);
if (txn->mt_parent) {
if (!txn->mt_parent) {
if (mdb_midl_shrink(&txn->mt_free_pgs))
env->me_free_pgs = txn->mt_free_pgs;
/* me_pgstate: */
env->me_pghead = NULL;
env->me_pglast = 0;
env->me_txn = NULL;
/* The writer mutex was locked in mdb_txn_begin. */
if (env->me_txns)
UNLOCK_MUTEX_W(env);
} else {
txn->mt_parent->mt_child = NULL;
env->me_pgstate = ((MDB_ntxn *)txn)->mnt_pgstate;
mdb_midl_free(txn->mt_free_pgs);
mdb_midl_free(txn->mt_spill_pgs);
free(txn->mt_u.dirty_list);
return;
}
if (mdb_midl_shrink(&txn->mt_free_pgs))
env->me_free_pgs = txn->mt_free_pgs;
env->me_pghead = NULL;
env->me_pglast = 0;
env->me_txn = NULL;
/* The writer mutex was locked in mdb_txn_begin. */
if (env->me_txns)
UNLOCK_MUTEX_W(env);
mdb_midl_free(pghead);
}
#ifdef VL32
{
unsigned i, n = txn->mt_rpages[0].mid;
for (i = 1; i <= n; i++) {
#ifdef _WIN32
UnmapViewOfFile(txn->mt_rpages[i].mptr);)
UnmapViewOfFile(txn->mt_rpages[i].mptr);
#else
MDB_page *mp = txn->mt_rpages[i].mptr;
int size = txn->mt_env->me_psize;
@ -3159,6 +3201,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
if (n) {
retry_write:
/* Write previous page(s) */
#ifdef MDB_USE_PWRITEV
wres = pwritev(env->me_fd, iov, n, wpos);
@ -3166,8 +3209,11 @@ mdb_page_flush(MDB_txn *txn, int keep)
if (n == 1) {
wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
} else {
retry_seek:
if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
rc = ErrCode();
if (rc == EINTR)
goto retry_seek;
DPRINTF(("lseek: %s", strerror(rc)));
return rc;
}
@ -3177,6 +3223,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
if (wres != wsize) {
if (wres < 0) {
rc = ErrCode();
if (rc == EINTR)
goto retry_write;
DPRINTF(("Write error: %s", strerror(rc)));
} else {
rc = EIO; /* TODO: Use which error code? */
@ -3546,7 +3594,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
int len;
#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \
len = pwrite(fd, ptr, size, pos); \
rc = (len >= 0); } while(0)
if (len == -1 && ErrCode() == EINTR) continue; \
rc = (len >= 0); break; } while(1)
#endif
DPUTS("writing new meta page");
@ -3651,6 +3700,7 @@ mdb_env_write_meta(MDB_txn *txn)
/* Write to the SYNC fd */
mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ?
env->me_fd : env->me_mfd;
retry_write:
#ifdef _WIN32
{
memset(&ov, 0, sizeof(ov));
@ -3663,6 +3713,8 @@ mdb_env_write_meta(MDB_txn *txn)
#endif
if (rc != len) {
rc = rc < 0 ? ErrCode() : EIO;
if (rc == EINTR)
goto retry_write;
DPUTS("write failed, disk error?");
/* On a failure, the pagecache still contains the new data.
* Write some old data back, to prevent it from being used.
@ -3894,6 +3946,32 @@ mdb_env_get_maxreaders(MDB_env *env, unsigned int *readers)
return MDB_SUCCESS;
}
static int ESECT
mdb_fsize(HANDLE fd, size_t *size)
{
#ifdef _WIN32
LARGE_INTEGER fsize;
if (!GetFileSizeEx(fd, &fsize))
return ErrCode();
*size = fsize.QuadPart;
#else
struct stat st;
if (fstat(fd, &st))
return ErrCode();
*size = st.st_size;
#endif
return MDB_SUCCESS;
}
#ifdef BROKEN_FDATASYNC
#include <sys/utsname.h>
#include <sys/vfs.h>
#endif
/** Further setup required for opening an LMDB environment
*/
static int ESECT
@ -3911,6 +3989,53 @@ mdb_env_open2(MDB_env *env)
else
env->me_pidquery = PROCESS_QUERY_INFORMATION;
#endif /* _WIN32 */
#ifdef BROKEN_FDATASYNC
/* ext3/ext4 fdatasync is broken on some older Linux kernels.
* https://lkml.org/lkml/2012/9/3/83
* Kernels after 3.6-rc6 are known good.
* https://lkml.org/lkml/2012/9/10/556
* See if the DB is on ext3/ext4, then check for new enough kernel
* Kernels 2.6.32.60, 2.6.34.15, 3.2.30, and 3.5.4 are also known
* to be patched.
*/
{
struct statfs st;
fstatfs(env->me_fd, &st);
while (st.f_type == 0xEF53) {
struct utsname uts;
int i;
uname(&uts);
if (uts.release[0] < '3') {
if (!strncmp(uts.release, "2.6.32.", 7)) {
i = atoi(uts.release+7);
if (i >= 60)
break; /* 2.6.32.60 and newer is OK */
} else if (!strncmp(uts.release, "2.6.34.", 7)) {
i = atoi(uts.release+7);
if (i >= 15)
break; /* 2.6.34.15 and newer is OK */
}
} else if (uts.release[0] == '3') {
i = atoi(uts.release+2);
if (i > 5)
break; /* 3.6 and newer is OK */
if (i == 5) {
i = atoi(uts.release+4);
if (i >= 4)
break; /* 3.5.4 and newer is OK */
} else if (i == 2) {
i = atoi(uts.release+4);
if (i >= 30)
break; /* 3.2.30 and newer is OK */
}
} else { /* 4.x and newer is OK */
break;
}
env->me_flags |= MDB_FSYNCONLY;
break;
}
}
#endif
memset(&meta, 0, sizeof(meta));
@ -4042,7 +4167,7 @@ PIMAGE_TLS_CALLBACK mdb_tls_cbp __attribute__((section (".CRT$XLB"))) = mdb_tls_
extern const PIMAGE_TLS_CALLBACK mdb_tls_cbp;
const PIMAGE_TLS_CALLBACK mdb_tls_cbp = mdb_tls_callback;
#pragma const_seg()
#else /* WIN32 */
#else /* _WIN32 */
#pragma comment(linker, "/INCLUDE:__tls_used")
#pragma comment(linker, "/INCLUDE:_mdb_tls_cbp")
#pragma data_seg(".CRT$XLB")
@ -4092,7 +4217,7 @@ mdb_env_share_locks(MDB_env *env, int *excl)
return rc;
}
/** Try to get exlusive lock, otherwise shared.
/** Try to get exclusive lock, otherwise shared.
* Maintain *excl = -1: no/unknown lock, 0: shared, 1: exclusive.
*/
static int ESECT
@ -4233,7 +4358,6 @@ mdb_hash_enc(MDB_val *val, char *encbuf)
* @param[in] env The LMDB environment.
* @param[in] lpath The pathname of the file used for the lock region.
* @param[in] mode The Unix permissions for the file, if we create it.
* @param[out] excl Resulting file lock type: -1 none, 0 shared, 1 exclusive
* @param[in,out] excl In -1, out lock type: -1 none, 0 shared, 1 exclusive
* @return 0 on success, non-zero on failure.
*/
@ -4599,7 +4723,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
if (!(flags & MDB_RDONLY)) {
MDB_txn *txn;
int tsize = sizeof(MDB_txn), size = tsize + env->me_maxdbs *
(sizeof(MDB_db)+sizeof(MDB_cursor)+sizeof(unsigned int)+1);
(sizeof(MDB_db)+sizeof(MDB_cursor *)+sizeof(unsigned int)+1);
txn = calloc(1, size);
if (txn) {
txn->mt_dbs = (MDB_db *)((char *)txn + tsize);
@ -4607,6 +4731,7 @@ mdb_env_open(MDB_env *env, const char *path, unsigned int flags, mdb_mode_t mode
txn->mt_dbiseqs = (unsigned int *)(txn->mt_cursors + env->me_maxdbs);
txn->mt_dbflags = (unsigned char *)(txn->mt_dbiseqs + env->me_maxdbs);
txn->mt_env = env;
txn->mt_dbxs = env->me_dbxs;
#ifdef VL32
txn->mt_rpages = calloc(MDB_IDL_UM_SIZE, sizeof(MDB_ID2));
if (!txn->mt_rpages) {
@ -4639,13 +4764,15 @@ mdb_env_close0(MDB_env *env, int excl)
return;
/* Doing this here since me_dbxs may not exist during mdb_env_close */
for (i = env->me_maxdbs; --i > MAIN_DBI; )
free(env->me_dbxs[i].md_name.mv_data);
if (env->me_dbxs) {
for (i = env->me_maxdbs; --i > MAIN_DBI; )
free(env->me_dbxs[i].md_name.mv_data);
free(env->me_dbxs);
}
free(env->me_pbuf);
free(env->me_dbiseqs);
free(env->me_dbflags);
free(env->me_dbxs);
free(env->me_path);
free(env->me_dirty_list);
free(env->me_txn0);
@ -4752,7 +4879,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
*(size_t *)a->mv_data > *(size_t *)b->mv_data;
}
/** Compare two items pointing at aligned unsigned int's */
/** Compare two items pointing at aligned unsigned int's.
*
* This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
* but #mdb_cmp_clong() is called instead if the data type is size_t.
*/
static int
mdb_cmp_int(const MDB_val *a, const MDB_val *b)
{
@ -4790,13 +4921,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
#endif
}
/** Compare two items pointing at size_t's of unknown alignment. */
#ifdef MISALIGNED_OK
# define mdb_cmp_clong mdb_cmp_long
#else
# define mdb_cmp_clong mdb_cmp_cint
#endif
/** Compare two items lexically */
static int
mdb_cmp_memn(const MDB_val *a, const MDB_val *b)
@ -5432,6 +5556,7 @@ mdb_cursor_sibling(MDB_cursor *mc, int move_right)
#ifdef VL32
op = mc->mc_pg[mc->mc_top];
#endif
mdb_cursor_pop(mc);
DPRINTF(("parent page is page %"Z"u, index %u",
mc->mc_pg[mc->mc_top]->mp_pgno, mc->mc_ki[mc->mc_top]));
@ -5590,11 +5715,11 @@ mdb_cursor_prev(MDB_cursor *mc, MDB_val *key, MDB_val *data, MDB_cursor_op op)
}
return rc;
}
} else {
mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
if (op == MDB_PREV_DUP)
return MDB_NOTFOUND;
}
} else {
mc->mc_xcursor->mx_cursor.mc_flags &= ~(C_INITIALIZED|C_EOF);
if (op == MDB_PREV_DUP)
return MDB_NOTFOUND;
}
}
@ -5806,15 +5931,21 @@ set1:
return rc;
}
} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
MDB_val d2;
if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS)
MDB_val olddata;
MDB_cmp_func *dcmp;
if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
return rc;
rc = mc->mc_dbx->md_dcmp(data, &d2);
dcmp = mc->mc_dbx->md_dcmp;
#if UINT_MAX < SIZE_MAX
if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
dcmp = mdb_cmp_clong;
#endif
rc = dcmp(data, &olddata);
if (rc) {
if (op == MDB_GET_BOTH || rc > 0)
return MDB_NOTFOUND;
rc = 0;
*data = d2;
*data = olddata;
}
} else {
@ -6324,16 +6455,17 @@ more:
/* Was a single item before, must convert now */
if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
MDB_cmp_func *dcmp;
/* Just overwrite the current item */
if (flags == MDB_CURRENT)
goto current;
dcmp = mc->mc_dbx->md_dcmp;
#if UINT_MAX < SIZE_MAX
if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
mc->mc_dbx->md_dcmp = mdb_cmp_clong;
if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
dcmp = mdb_cmp_clong;
#endif
/* does data match? */
if (!mc->mc_dbx->md_dcmp(data, &olddata)) {
if (!dcmp(data, &olddata)) {
if (flags & MDB_NODUPDATA)
return MDB_KEYEXIST;
/* overwrite it */
@ -7153,6 +7285,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
mc->mc_snum = 0;
mc->mc_top = 0;
mc->mc_pg[0] = 0;
mc->mc_ki[0] = 0;
mc->mc_flags = 0;
if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
mdb_tassert(txn, mx != NULL);
@ -7526,7 +7659,7 @@ mdb_node_move(MDB_cursor *csrc, MDB_cursor *cdst)
cdst->mc_ki[cdst->mc_top] = 0;
rc = mdb_update_key(cdst, &nullkey);
cdst->mc_ki[cdst->mc_top] = ix;
mdb_cassert(csrc, rc == MDB_SUCCESS);
mdb_cassert(cdst, rc == MDB_SUCCESS);
}
}
@ -7782,12 +7915,12 @@ mdb_rebalance(MDB_cursor *mc)
m3 = m2;
if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
if (m3->mc_pg[0] == mp) {
m3->mc_snum--;
m3->mc_top--;
for (i=0; i<m3->mc_snum; i++) {
m3->mc_pg[i] = m3->mc_pg[i+1];
m3->mc_ki[i] = m3->mc_ki[i+1];
}
m3->mc_snum--;
m3->mc_top--;
}
}
}
@ -7855,9 +7988,23 @@ mdb_rebalance(MDB_cursor *mc)
if (mc->mc_ki[ptop] == 0) {
rc = mdb_page_merge(&mn, mc);
} else {
MDB_cursor dummy;
oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
/* We want mdb_rebalance to find mn when doing fixups */
if (mc->mc_flags & C_SUB) {
dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
dummy.mc_xcursor = (MDB_xcursor *)&mn;
} else {
mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
}
rc = mdb_page_merge(mc, &mn);
if (mc->mc_flags & C_SUB)
mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next;
else
mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
mdb_cursor_copy(&mn, mc);
}
mc->mc_flags &= ~C_EOF;
@ -7884,6 +8031,13 @@ mdb_cursor_del0(MDB_cursor *mc)
MDB_cursor *m2, *m3;
MDB_dbi dbi = mc->mc_dbi;
/* DB is totally empty now, just bail out.
* Other cursors adjustments were already done
* by mdb_rebalance and aren't needed here.
*/
if (!mc->mc_snum)
return rc;
mp = mc->mc_pg[mc->mc_top];
nkeys = NUMKEYS(mp);
@ -8751,8 +8905,12 @@ mdb_env_copyfd1(MDB_env *env, HANDLE fd)
/* Set metapage 1 */
mm->mm_last_pg = txn->mt_next_pgno - freecount - 1;
mm->mm_dbs[1] = txn->mt_dbs[1];
mm->mm_dbs[1].md_root = mm->mm_last_pg;
mm->mm_txnid = 1;
if (mm->mm_last_pg > 1) {
mm->mm_dbs[1].md_root = mm->mm_last_pg;
mm->mm_txnid = 1;
} else {
mm->mm_dbs[1].md_root = P_INVALID;
}
}
my.mc_wlen[0] = env->me_psize * 2;
my.mc_txn = txn;
@ -8847,21 +9005,13 @@ mdb_env_copyfd0(MDB_env *env, HANDLE fd)
goto leave;
w2 = txn->mt_next_pgno * env->me_psize;
#ifdef WIN32
{
LARGE_INTEGER fsize;
GetFileSizeEx(env->me_fd, &fsize);
if (w2 > fsize.QuadPart)
w2 = fsize.QuadPart;
size_t fsize = 0;
if ((rc = mdb_fsize(env->me_fd, &fsize)))
goto leave;
if (w2 > fsize)
w2 = fsize;
}
#else
{
struct stat st;
fstat(env->me_fd, &st);
if (w2 > (size_t)st.st_size)
w2 = st.st_size;
}
#endif
wsize = w2 - wsize;
while (wsize > 0) {
if (wsize > MAX_WRITE)

2
external/db_drivers/liblmdb32/mdb_copy.1 vendored
View File

@ -1,5 +1,5 @@
.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_copy \- LMDB environment copy tool

4
external/db_drivers/liblmdb32/mdb_copy.c vendored
View File

@ -1,6 +1,6 @@
/* mdb_copy.c - memory-mapped database backup tool */
/*
* Copyright 2012 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -64,7 +64,7 @@ int main(int argc,char * argv[])
act = "opening environment";
rc = mdb_env_create(&env);
if (rc == MDB_SUCCESS) {
rc = mdb_env_open(env, argv[1], flags, 0664);
rc = mdb_env_open(env, argv[1], flags, 0600);
}
if (rc == MDB_SUCCESS) {
act = "copying";

2
external/db_drivers/liblmdb32/mdb_dump.1 vendored
View File

@ -1,5 +1,5 @@
.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_dump \- LMDB environment export tool

2
external/db_drivers/liblmdb32/mdb_dump.c vendored
View File

@ -1,6 +1,6 @@
/* mdb_dump.c - memory-mapped database dump tool */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
external/db_drivers/liblmdb32/mdb_load.1 vendored
View File

@ -1,5 +1,5 @@
.TH MDB_LOAD 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2014-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_load \- LMDB environment import tool

12
external/db_drivers/liblmdb32/mdb_load.c vendored
View File

@ -1,6 +1,6 @@
/* mdb_load.c - memory-mapped database load tool */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -176,7 +176,7 @@ static int unhex(unsigned char *c2)
static int readline(MDB_val *out, MDB_val *buf)
{
unsigned char *c1, *c2, *end;
size_t len;
size_t len, l2;
int c;
if (!(mode & NOHDR)) {
@ -206,6 +206,7 @@ badend:
c1 = buf->mv_data;
len = strlen((char *)c1);
l2 = len;
/* Is buffer too short? */
while (c1[len-1] != '\n') {
@ -217,17 +218,18 @@ badend:
return EOF;
}
c1 = buf->mv_data;
c1 += buf->mv_size;
if (fgets((char *)c1, buf->mv_size, stdin) == NULL) {
c1 += l2;
if (fgets((char *)c1, buf->mv_size+1, stdin) == NULL) {
Eof = 1;
badend();
return EOF;
}
buf->mv_size *= 2;
len = strlen((char *)c1);
l2 += len;
}
c1 = c2 = buf->mv_data;
len = strlen((char *)c1);
len = l2;
c1[--len] = '\0';
end = c1 + len;

2
external/db_drivers/liblmdb32/mdb_stat.1 vendored
View File

@ -1,5 +1,5 @@
.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14"
.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copyright 2012-2015 Howard Chu, Symas Corp. All Rights Reserved.
.\" Copying restrictions apply. See COPYRIGHT/LICENSE.
.SH NAME
mdb_stat \- LMDB environment status tool

2
external/db_drivers/liblmdb32/mdb_stat.c vendored
View File

@ -1,6 +1,6 @@
/* mdb_stat.c - memory-mapped database status tool */
/*
* Copyright 2011-2013 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

2
external/db_drivers/liblmdb32/midl.c vendored
View File

@ -3,7 +3,7 @@
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2000-2014 The OpenLDAP Foundation.
* Copyright 2000-2015 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

4
external/db_drivers/liblmdb32/midl.h vendored
View File

@ -11,7 +11,7 @@
/* $OpenLDAP$ */
/* This work is part of OpenLDAP Software <http://www.openldap.org/>.
*
* Copyright 2000-2014 The OpenLDAP Foundation.
* Copyright 2000-2015 The OpenLDAP Foundation.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -58,7 +58,7 @@ typedef MDB_ID *MDB_IDL;
#ifdef VL32
#define MDB_IDL_LOGN 10 /* DB_SIZE is 2^10, UM_SIZE is 2^11 */
#else
#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
#define MDB_IDL_LOGN 16 /* DB_SIZE is 2^16, UM_SIZE is 2^17 */
#endif
#define MDB_IDL_DB_SIZE (1<<MDB_IDL_LOGN)
#define MDB_IDL_UM_SIZE (1<<(MDB_IDL_LOGN+1))

20
external/db_drivers/liblmdb32/mtest.c vendored
View File

@ -1,6 +1,6 @@
/* mtest.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -45,19 +45,22 @@ int main(int argc,char * argv[])
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_open(env, "./testdb", 0 /* MDB_FIXEDMAP |MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, NULL, 0, &dbi));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
@ -68,7 +71,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -97,7 +100,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -128,6 +131,7 @@ int main(int argc,char * argv[])
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
@ -164,9 +168,9 @@ int main(int argc,char * argv[])
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;

19
external/db_drivers/liblmdb32/mtest2.c vendored
View File

@ -1,6 +1,6 @@
/* mtest2.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -47,20 +47,22 @@ int main(int argc,char * argv[])
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
@ -68,7 +70,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -97,7 +99,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -114,10 +116,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

14
external/db_drivers/liblmdb32/mtest3.c vendored
View File

@ -1,6 +1,6 @@
/* mtest3.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -53,8 +53,9 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
@ -73,7 +74,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -107,7 +108,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -124,10 +125,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

14
external/db_drivers/liblmdb32/mtest4.c vendored
View File

@ -1,6 +1,6 @@
/* mtest4.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -51,8 +51,9 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
@ -72,7 +73,7 @@ int main(int argc,char * argv[])
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -142,7 +143,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -159,10 +160,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

14
external/db_drivers/liblmdb32/mtest5.c vendored
View File

@ -1,6 +1,6 @@
/* mtest5.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -53,8 +53,9 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
key.mv_size = sizeof(int);
@ -75,7 +76,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -109,7 +110,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -126,10 +127,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

20
external/db_drivers/liblmdb32/mtest6.c vendored
View File

@ -1,6 +1,6 @@
/* mtest6.c - memory-mapped database tester/toy */
/*
* Copyright 2011-2014 Howard Chu, Symas Corp.
* Copyright 2011-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -31,7 +31,7 @@ int main(int argc,char * argv[])
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_val key, data, sdata;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
@ -46,33 +46,37 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
data.mv_size = mst.ms_psize / 4 - 30;
data.mv_data = sval;
sdata.mv_size = mst.ms_psize / 4 - 30;
sdata.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
@ -110,7 +114,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -127,9 +131,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
#endif
mdb_env_close(env);

2
external/db_drivers/liblmdb32/sample-bdb.txt vendored
View File

@ -3,7 +3,7 @@
* Do a line-by-line comparison of this and sample-mdb.txt
*/
/*
* Copyright 2012 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without

6
external/db_drivers/liblmdb32/sample-mdb.txt vendored
View File

@ -3,7 +3,7 @@
* Do a line-by-line comparison of this and sample-bdb.txt
*/
/*
* Copyright 2012 Howard Chu, Symas Corp.
* Copyright 2012-2015 Howard Chu, Symas Corp.
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
@ -32,7 +32,7 @@ int main(int argc,char * argv[])
rc = mdb_env_create(&env);
rc = mdb_env_open(env, "./testdb", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, 0, &dbi);
rc = mdb_dbi_open(txn, NULL, 0, &dbi);
key.mv_size = sizeof(int);
key.mv_data = sval;
@ -56,7 +56,7 @@ int main(int argc,char * argv[])
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_close(env, dbi);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

17
external/db_drivers/liblmdb64/CHANGES vendored
View File

@ -1,18 +1,31 @@
LMDB 0.9 Change Log
LMDB 0.9.15 Release Engineering
LMDB 0.9.15 Release (2015/06/19)
Fix txn init (ITS#7961,#7987)
Fix MDB_PREV_DUP (ITS#7955,#7671)
Fix compact of empty env (ITS#7956)
Fix mdb_copy file mode
Fix mdb_env_close() after failed mdb_env_open()
Fix mdb_rebalance collapsing root (ITS#8062)
Fix mdb_load with large values (ITS#8066)
Fix to retry writes on EINTR (ITS#8106)
Fix mdb_cursor_del on empty DB (ITS#8109)
Fix MDB_INTEGERDUP key compare (ITS#8117)
Fix error handling (ITS#7959,#8157,etc.)
Fix race conditions (ITS#7969,7970)
Added workaround for fdatasync bug in ext3fs
Build
Don't use -fPIC for static lib
Update .gitignore (ITS#7952,#7953)
Cleanup for "make test" (ITS#7841)
Cleanup for "make test" (ITS#7841), "make clean", mtest*.c
Misc. Android/Windows cleanup
Documentation
Fix MDB_APPEND doc
Fix MDB_MAXKEYSIZE doc (ITS#8156)
Fix mdb_cursor_put,mdb_cursor_del EACCES description
Fix mdb_env_sync(MDB_RDONLY env) doc (ITS#8021)
Clarify MDB_WRITEMAP doc (ITS#8021)
Clarify mdb_env_open doc
Clarify mdb_dbi_open doc
LMDB 0.9.14 Release (2014/09/20)

4
external/db_drivers/liblmdb64/CMakeLists.txt vendored
View File

@ -26,6 +26,10 @@
# STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
# THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
if(FREEBSD)
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DMDB_DSYNC=O_SYNC")
endif()
set (lmdb_sources
mdb.c
midl.c)

38
external/db_drivers/liblmdb64/lmdb.h vendored
View File

@ -184,7 +184,7 @@ typedef int mdb_filehandle_t;
/** Library minor version */
#define MDB_VERSION_MINOR 9
/** Library patch version */
#define MDB_VERSION_PATCH 14
#define MDB_VERSION_PATCH 15
/** Combine args a,b,c into a single integer for easy version comparisons */
#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c))
@ -194,7 +194,7 @@ typedef int mdb_filehandle_t;
MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
/** The release date of this library version */
#define MDB_VERSION_DATE "September 20, 2014"
#define MDB_VERSION_DATE "June 19, 2015"
/** A stringifier for the version info */
#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")"
@ -296,12 +296,12 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
#define MDB_REVERSEKEY 0x02
/** use sorted duplicates */
#define MDB_DUPSORT 0x04
/** numeric keys in native byte order.
/** numeric keys in native byte order: either unsigned int or size_t.
* The keys must all be of the same size. */
#define MDB_INTEGERKEY 0x08
/** with #MDB_DUPSORT, sorted dup items have fixed size */
#define MDB_DUPFIXED 0x10
/** with #MDB_DUPSORT, dups are numeric in native byte order */
/** with #MDB_DUPSORT, dups are #MDB_INTEGERKEY-style integers */
#define MDB_INTEGERDUP 0x20
/** with #MDB_DUPSORT, use reverse string dups */
#define MDB_REVERSEDUP 0x40
@ -588,8 +588,8 @@ int mdb_env_create(MDB_env **env);
* reserved in that case.
* This flag may be changed at any time using #mdb_env_set_flags().
* </ul>
* @param[in] mode The UNIX permissions to set on created files. This parameter
* is ignored on Windows.
* @param[in] mode The UNIX permissions to set on created files and semaphores.
* This parameter is ignored on Windows.
* @return A non-zero error value on failure and 0 on success. Some possible
* errors are:
* <ul>
@ -1021,14 +1021,16 @@ int mdb_txn_renew(MDB_txn *txn);
* The database handle may be discarded by calling #mdb_dbi_close().
* The old database handle is returned if the database was already open.
* The handle may only be closed once.
*
* The database handle will be private to the current transaction until
* the transaction is successfully committed. If the transaction is
* aborted the handle will be closed automatically.
* After a successful commit the
* handle will reside in the shared environment, and may be used
* by other transactions. This function must not be called from
* multiple concurrent transactions in the same process. A transaction
* that uses this function must finish (either commit or abort) before
* After a successful commit the handle will reside in the shared
* environment, and may be used by other transactions.
*
* This function must not be called from multiple concurrent
* transactions in the same process. A transaction that uses
* this function must finish (either commit or abort) before
* any other transaction in the process may use this function.
*
* To use named databases (with name != NULL), #mdb_env_set_maxdbs()
@ -1050,9 +1052,9 @@ int mdb_txn_renew(MDB_txn *txn);
* keys may have multiple data items, stored in sorted order.) By default
* keys must be unique and may have only a single data item.
* <li>#MDB_INTEGERKEY
* Keys are binary integers in native byte order. Setting this option
* requires all keys to be the same size, typically sizeof(int)
* or sizeof(size_t).
* Keys are binary integers in native byte order, either unsigned int
* or size_t, and will be sorted as such.
* The keys must all be of the same size.
* <li>#MDB_DUPFIXED
* This flag may only be used in combination with #MDB_DUPSORT. This option
* tells the library that the data items for this database are all the same
@ -1060,8 +1062,8 @@ int mdb_txn_renew(MDB_txn *txn);
* all data items are the same size, the #MDB_GET_MULTIPLE and #MDB_NEXT_MULTIPLE
* cursor operations may be used to retrieve multiple items at once.
* <li>#MDB_INTEGERDUP
* This option specifies that duplicate data items are also integers, and
* should be sorted as such.
* This option specifies that duplicate data items are binary integers,
* similar to #MDB_INTEGERKEY keys.
* <li>#MDB_REVERSEDUP
* This option specifies that duplicate data items should be compared as
* strings in reverse order.
@ -1450,7 +1452,7 @@ int mdb_cursor_get(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
* <ul>
* <li>#MDB_MAP_FULL - the database is full, see #mdb_env_set_mapsize().
* <li>#MDB_TXN_FULL - the transaction has too many dirty pages.
* <li>EACCES - an attempt was made to modify a read-only database.
* <li>EACCES - an attempt was made to write in a read-only transaction.
* <li>EINVAL - an invalid parameter was specified.
* </ul>
*/
@ -1470,7 +1472,7 @@ int mdb_cursor_put(MDB_cursor *cursor, MDB_val *key, MDB_val *data,
* @return A non-zero error value on failure and 0 on success. Some possible
* errors are:
* <ul>
* <li>EACCES - an attempt was made to modify a read-only database.
* <li>EACCES - an attempt was made to write in a read-only transaction.
* <li>EINVAL - an invalid parameter was specified.
* </ul>
*/

118
external/db_drivers/liblmdb64/mdb.c vendored
View File

@ -446,12 +446,17 @@ static txnid_t mdb_debug_start;
/** The version number for a database's lockfile format. */
#define MDB_LOCK_VERSION 1
/** @brief The max size of a key we can write, or 0 for dynamic max.
/** @brief The max size of a key we can write, or 0 for computed max.
*
* Define this as 0 to compute the max from the page size. 511
* is default for backwards compat: liblmdb <= 0.9.10 can break
* when modifying a DB with keys/dupsort data bigger than its max.
* #MDB_DEVEL sets the default to 0.
* This macro should normally be left alone or set to 0.
* Note that a database with big keys or dupsort data cannot be
* reliably modified by a liblmdb which uses a smaller max.
* The default is 511 for backwards compat, or 0 when #MDB_DEVEL.
*
* Other values are allowed, for backwards compat. However:
* A value bigger than the computed max can break if you do not
* know what you are doing, and liblmdb <= 0.9.10 can break when
* modifying a DB with keys/dupsort data bigger than its max.
*
* Data items in an #MDB_DUPSORT database are also limited to
* this size, since they're actually keys of a sub-DB. Keys and
@ -1245,6 +1250,13 @@ static void mdb_default_cmp(MDB_txn *txn, MDB_dbi dbi);
static MDB_cmp_func mdb_cmp_memn, mdb_cmp_memnr, mdb_cmp_int, mdb_cmp_cint, mdb_cmp_long;
/** @endcond */
/** Compare two items pointing at size_t's of unknown alignment. */
#ifdef MISALIGNED_OK
# define mdb_cmp_clong mdb_cmp_long
#else
# define mdb_cmp_clong mdb_cmp_cint
#endif
#ifdef _WIN32
static SECURITY_DESCRIPTOR mdb_null_sd;
static SECURITY_ATTRIBUTES mdb_all_sa;
@ -1558,7 +1570,12 @@ mdb_cmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
int
mdb_dcmp(MDB_txn *txn, MDB_dbi dbi, const MDB_val *a, const MDB_val *b)
{
return txn->mt_dbxs[dbi].md_dcmp(a, b);
MDB_cmp_func *dcmp = txn->mt_dbxs[dbi].md_dcmp;
#if UINT_MAX < SIZE_MAX
if (dcmp == mdb_cmp_int && a->mv_size == sizeof(size_t))
dcmp = mdb_cmp_clong;
#endif
return dcmp(a, b);
}
/** Allocate memory for a page.
@ -2486,14 +2503,11 @@ mdb_txn_renew0(MDB_txn *txn)
MDB_env *env = txn->mt_env;
MDB_txninfo *ti = env->me_txns;
MDB_meta *meta;
unsigned int i, nr;
unsigned int i, nr, flags = txn->mt_flags;
uint16_t x;
int rc, new_notls = 0;
if (txn->mt_flags & MDB_TXN_RDONLY) {
/* Setup db info */
txn->mt_numdbs = env->me_numdbs;
txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */
if ((flags &= MDB_TXN_RDONLY) != 0) {
if (!ti) {
meta = env->me_metas[ mdb_env_pick_meta(env) ];
txn->mt_txnid = meta->mm_txnid;
@ -2546,6 +2560,7 @@ mdb_txn_renew0(MDB_txn *txn)
txn->mt_u.reader = r;
meta = env->me_metas[txn->mt_txnid & 1];
}
txn->mt_dbxs = env->me_dbxs; /* mostly static anyway */
} else {
if (ti) {
LOCK_MUTEX_W(env);
@ -2556,14 +2571,11 @@ mdb_txn_renew0(MDB_txn *txn)
meta = env->me_metas[ mdb_env_pick_meta(env) ];
txn->mt_txnid = meta->mm_txnid;
}
/* Setup db info */
txn->mt_numdbs = env->me_numdbs;
txn->mt_txnid++;
#if MDB_DEBUG
if (txn->mt_txnid == mdb_debug_start)
mdb_debug = 1;
#endif
txn->mt_flags = 0;
txn->mt_child = NULL;
txn->mt_loose_pgs = NULL;
txn->mt_loose_count = 0;
@ -2583,6 +2595,10 @@ mdb_txn_renew0(MDB_txn *txn)
/* Moved to here to avoid a data race in read TXNs */
txn->mt_next_pgno = meta->mm_last_pg+1;
txn->mt_flags = flags;
/* Setup db info */
txn->mt_numdbs = env->me_numdbs;
for (i=2; i<txn->mt_numdbs; i++) {
x = env->me_dbflags[i];
txn->mt_dbs[i].md_flags = x & PERSISTENT_FLAGS;
@ -3132,6 +3148,7 @@ mdb_page_flush(MDB_txn *txn, int keep)
/* Write up to MDB_COMMIT_PAGES dirty pages at a time. */
if (pos!=next_pos || n==MDB_COMMIT_PAGES || wsize+size>MAX_WRITE) {
if (n) {
retry_write:
/* Write previous page(s) */
#ifdef MDB_USE_PWRITEV
wres = pwritev(env->me_fd, iov, n, wpos);
@ -3139,8 +3156,11 @@ mdb_page_flush(MDB_txn *txn, int keep)
if (n == 1) {
wres = pwrite(env->me_fd, iov[0].iov_base, wsize, wpos);
} else {
retry_seek:
if (lseek(env->me_fd, wpos, SEEK_SET) == -1) {
rc = ErrCode();
if (rc == EINTR)
goto retry_seek;
DPRINTF(("lseek: %s", strerror(rc)));
return rc;
}
@ -3150,6 +3170,8 @@ mdb_page_flush(MDB_txn *txn, int keep)
if (wres != wsize) {
if (wres < 0) {
rc = ErrCode();
if (rc == EINTR)
goto retry_write;
DPRINTF(("Write error: %s", strerror(rc)));
} else {
rc = EIO; /* TODO: Use which error code? */
@ -3519,7 +3541,8 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
int len;
#define DO_PWRITE(rc, fd, ptr, size, len, pos) do { \
len = pwrite(fd, ptr, size, pos); \
rc = (len >= 0); } while(0)
if (len == -1 && ErrCode() == EINTR) continue; \
rc = (len >= 0); break; } while(1)
#endif
DPUTS("writing new meta page");
@ -3624,6 +3647,7 @@ mdb_env_write_meta(MDB_txn *txn)
/* Write to the SYNC fd */
mfd = env->me_flags & (MDB_NOSYNC|MDB_NOMETASYNC) ?
env->me_fd : env->me_mfd;
retry_write:
#ifdef _WIN32
{
memset(&ov, 0, sizeof(ov));
@ -3636,6 +3660,8 @@ mdb_env_write_meta(MDB_txn *txn)
#endif
if (rc != len) {
rc = rc < 0 ? ErrCode() : EIO;
if (rc == EINTR)
goto retry_write;
DPUTS("write failed, disk error?");
/* On a failure, the pagecache still contains the new data.
* Write some old data back, to prevent it from being used.
@ -4761,7 +4787,11 @@ mdb_cmp_long(const MDB_val *a, const MDB_val *b)
*(size_t *)a->mv_data > *(size_t *)b->mv_data;
}
/** Compare two items pointing at aligned unsigned int's */
/** Compare two items pointing at aligned unsigned int's.
*
* This is also set as #MDB_INTEGERDUP|#MDB_DUPFIXED's #MDB_dbx.%md_dcmp,
* but #mdb_cmp_clong() is called instead if the data type is size_t.
*/
static int
mdb_cmp_int(const MDB_val *a, const MDB_val *b)
{
@ -4799,13 +4829,6 @@ mdb_cmp_cint(const MDB_val *a, const MDB_val *b)
#endif
}
/** Compare two items pointing at size_t's of unknown alignment. */
#ifdef MISALIGNED_OK
# define mdb_cmp_clong mdb_cmp_long
#else
# define mdb_cmp_clong mdb_cmp_cint
#endif
/** Compare two items lexically */
static int
mdb_cmp_memn(const MDB_val *a, const MDB_val *b)
@ -5741,15 +5764,21 @@ set1:
return rc;
}
} else if (op == MDB_GET_BOTH || op == MDB_GET_BOTH_RANGE) {
MDB_val d2;
if ((rc = mdb_node_read(mc->mc_txn, leaf, &d2)) != MDB_SUCCESS)
MDB_val olddata;
MDB_cmp_func *dcmp;
if ((rc = mdb_node_read(mc->mc_txn, leaf, &olddata)) != MDB_SUCCESS)
return rc;
rc = mc->mc_dbx->md_dcmp(data, &d2);
dcmp = mc->mc_dbx->md_dcmp;
#if UINT_MAX < SIZE_MAX
if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
dcmp = mdb_cmp_clong;
#endif
rc = dcmp(data, &olddata);
if (rc) {
if (op == MDB_GET_BOTH || rc > 0)
return MDB_NOTFOUND;
rc = 0;
*data = d2;
*data = olddata;
}
} else {
@ -6259,16 +6288,17 @@ more:
/* Was a single item before, must convert now */
if (!F_ISSET(leaf->mn_flags, F_DUPDATA)) {
MDB_cmp_func *dcmp;
/* Just overwrite the current item */
if (flags == MDB_CURRENT)
goto current;
dcmp = mc->mc_dbx->md_dcmp;
#if UINT_MAX < SIZE_MAX
if (mc->mc_dbx->md_dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
mc->mc_dbx->md_dcmp = mdb_cmp_clong;
if (dcmp == mdb_cmp_int && olddata.mv_size == sizeof(size_t))
dcmp = mdb_cmp_clong;
#endif
/* does data match? */
if (!mc->mc_dbx->md_dcmp(data, &olddata)) {
if (!dcmp(data, &olddata)) {
if (flags & MDB_NODUPDATA)
return MDB_KEYEXIST;
/* overwrite it */
@ -7088,6 +7118,7 @@ mdb_cursor_init(MDB_cursor *mc, MDB_txn *txn, MDB_dbi dbi, MDB_xcursor *mx)
mc->mc_snum = 0;
mc->mc_top = 0;
mc->mc_pg[0] = 0;
mc->mc_ki[0] = 0;
mc->mc_flags = 0;
if (txn->mt_dbs[dbi].md_flags & MDB_DUPSORT) {
mdb_tassert(txn, mx != NULL);
@ -7717,12 +7748,12 @@ mdb_rebalance(MDB_cursor *mc)
m3 = m2;
if (m3 == mc || m3->mc_snum < mc->mc_snum) continue;
if (m3->mc_pg[0] == mp) {
m3->mc_snum--;
m3->mc_top--;
for (i=0; i<m3->mc_snum; i++) {
m3->mc_pg[i] = m3->mc_pg[i+1];
m3->mc_ki[i] = m3->mc_ki[i+1];
}
m3->mc_snum--;
m3->mc_top--;
}
}
}
@ -7790,9 +7821,23 @@ mdb_rebalance(MDB_cursor *mc)
if (mc->mc_ki[ptop] == 0) {
rc = mdb_page_merge(&mn, mc);
} else {
MDB_cursor dummy;
oldki += NUMKEYS(mn.mc_pg[mn.mc_top]);
mn.mc_ki[mn.mc_top] += mc->mc_ki[mn.mc_top] + 1;
/* We want mdb_rebalance to find mn when doing fixups */
if (mc->mc_flags & C_SUB) {
dummy.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
mc->mc_txn->mt_cursors[mc->mc_dbi] = &dummy;
dummy.mc_xcursor = (MDB_xcursor *)&mn;
} else {
mn.mc_next = mc->mc_txn->mt_cursors[mc->mc_dbi];
mc->mc_txn->mt_cursors[mc->mc_dbi] = &mn;
}
rc = mdb_page_merge(mc, &mn);
if (mc->mc_flags & C_SUB)
mc->mc_txn->mt_cursors[mc->mc_dbi] = dummy.mc_next;
else
mc->mc_txn->mt_cursors[mc->mc_dbi] = mn.mc_next;
mdb_cursor_copy(&mn, mc);
}
mc->mc_flags &= ~C_EOF;
@ -7819,6 +7864,13 @@ mdb_cursor_del0(MDB_cursor *mc)
MDB_cursor *m2, *m3;
MDB_dbi dbi = mc->mc_dbi;
/* DB is totally empty now, just bail out.
* Other cursors adjustments were already done
* by mdb_rebalance and aren't needed here.
*/
if (!mc->mc_snum)
return rc;
mp = mc->mc_pg[mc->mc_top];
nkeys = NUMKEYS(mp);

18
external/db_drivers/liblmdb64/mtest.c vendored
View File

@ -45,19 +45,22 @@ int main(int argc,char * argv[])
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP /*|MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, NULL, 0, &dbi));
E(mdb_dbi_open(txn, NULL, 0, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
/* Set <data> in each iteration, since MDB_NOOVERWRITE may modify it */
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE))) {
j++;
data.mv_size = sizeof(sval);
@ -68,7 +71,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -97,7 +100,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -128,6 +131,7 @@ int main(int argc,char * argv[])
(int) key.mv_size, (char *) key.mv_data,
(int) data.mv_size, (char *) data.mv_data);
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
printf("Deleting with cursor\n");
@ -164,9 +168,9 @@ int main(int argc,char * argv[])
data.mv_data, (int) data.mv_size, (char *) data.mv_data);
}
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;

17
external/db_drivers/liblmdb64/mtest2.c vendored
View File

@ -47,20 +47,22 @@ int main(int argc,char * argv[])
}
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id1", MDB_CREATE, &dbi));
E(mdb_dbi_open(txn, "id1", MDB_CREATE, &dbi));
key.mv_size = sizeof(int);
key.mv_data = sval;
data.mv_size = sizeof(sval);
data.mv_data = sval;
printf("Adding %d values\n", count);
for (i=0;i<count;i++) {
sprintf(sval, "%03x %d foo bar", values[i], values[i]);
data.mv_size = sizeof(sval);
data.mv_data = sval;
if (RES(MDB_KEYEXIST, mdb_put(txn, dbi, &key, &data, MDB_NOOVERWRITE)))
j++;
}
@ -68,7 +70,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -97,7 +99,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -114,10 +116,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

12
external/db_drivers/liblmdb64/mtest3.c vendored
View File

@ -53,8 +53,9 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
@ -73,7 +74,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -107,7 +108,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -124,10 +125,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

12
external/db_drivers/liblmdb64/mtest4.c vendored
View File

@ -51,8 +51,9 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
E(mdb_dbi_open(txn, "id4", MDB_CREATE|MDB_DUPSORT|MDB_DUPFIXED, &dbi));
key.mv_size = sizeof(int);
key.mv_data = kval;
@ -72,7 +73,7 @@ int main(int argc,char * argv[])
/* there should be one full page of dups now.
*/
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -142,7 +143,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -159,10 +160,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

12
external/db_drivers/liblmdb64/mtest5.c vendored
View File

@ -53,8 +53,9 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_dbi_open(txn, "id2", MDB_CREATE|MDB_DUPSORT, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
key.mv_size = sizeof(int);
@ -75,7 +76,7 @@ int main(int argc,char * argv[])
E(mdb_txn_commit(txn));
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
printf("key: %p %.*s, data: %p %.*s\n",
@ -109,7 +110,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -126,10 +127,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_env_close(env);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

18
external/db_drivers/liblmdb64/mtest6.c vendored
View File

@ -31,7 +31,7 @@ int main(int argc,char * argv[])
int i = 0, j = 0, rc;
MDB_env *env;
MDB_dbi dbi;
MDB_val key, data;
MDB_val key, data, sdata;
MDB_txn *txn;
MDB_stat mst;
MDB_cursor *cursor;
@ -46,33 +46,37 @@ int main(int argc,char * argv[])
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_maxdbs(env, 4));
E(mdb_env_open(env, "./testdb", MDB_FIXEDMAP|MDB_NOSYNC, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));
E(mdb_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_dbi_open(txn, "id6", MDB_CREATE|MDB_INTEGERKEY, &dbi));
E(mdb_cursor_open(txn, dbi, &cursor));
E(mdb_stat(txn, dbi, &mst));
sval = calloc(1, mst.ms_psize / 4);
key.mv_size = sizeof(long);
key.mv_data = &kval;
data.mv_size = mst.ms_psize / 4 - 30;
data.mv_data = sval;
sdata.mv_size = mst.ms_psize / 4 - 30;
sdata.mv_data = sval;
printf("Adding 12 values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+4;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
printf("Adding 12 more values, should yield 3 splits\n");
for (i=0;i<12;i++) {
kval = i*5+1;
sprintf(sval, "%08x", kval);
data = sdata;
(void)RES(MDB_KEYEXIST, mdb_cursor_put(cursor, &key, &data, MDB_NOOVERWRITE));
}
E(mdb_cursor_get(cursor, &key, &data, MDB_FIRST));
@ -110,7 +114,7 @@ int main(int argc,char * argv[])
printf("Deleted %d values\n", j);
E(mdb_env_stat(env, &mst));
E(mdb_txn_begin(env, NULL, 1, &txn));
E(mdb_txn_begin(env, NULL, MDB_RDONLY, &txn));
E(mdb_cursor_open(txn, dbi, &cursor));
printf("Cursor next\n");
while ((rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT)) == 0) {
@ -127,9 +131,9 @@ int main(int argc,char * argv[])
}
CHECK(rc == MDB_NOTFOUND, "mdb_cursor_get");
mdb_cursor_close(cursor);
mdb_close(env, dbi);
mdb_txn_abort(txn);
mdb_dbi_close(env, dbi);
#endif
mdb_env_close(env);

4
external/db_drivers/liblmdb64/sample-mdb.txt vendored
View File

@ -32,7 +32,7 @@ int main(int argc,char * argv[])
rc = mdb_env_create(&env);
rc = mdb_env_open(env, "./testdb", 0, 0664);
rc = mdb_txn_begin(env, NULL, 0, &txn);
rc = mdb_open(txn, NULL, 0, &dbi);
rc = mdb_dbi_open(txn, NULL, 0, &dbi);
key.mv_size = sizeof(int);
key.mv_data = sval;
@ -56,7 +56,7 @@ int main(int argc,char * argv[])
mdb_cursor_close(cursor);
mdb_txn_abort(txn);
leave:
mdb_close(env, dbi);
mdb_dbi_close(env, dbi);
mdb_env_close(env);
return 0;
}

4
external/unbound/Makefile.in vendored
View File

@ -168,7 +168,7 @@ HOST_OBJ=unbound-host.lo
HOST_OBJ_LINK=$(HOST_OBJ) $(SLDNS_OBJ) $(COMPAT_OBJ_WITHOUT_CTIMEARC4) @WIN_HOST_OBJ_LINK@
UBANCHOR_SRC=smallapp/unbound-anchor.c
UBANCHOR_OBJ=unbound-anchor.lo
UBANCHOR_OBJ_LINK=$(UBANCHOR_OBJ) \
UBANCHOR_OBJ_LINK=$(UBANCHOR_OBJ) parseutil.lo \
$(COMPAT_OBJ_WITHOUT_CTIME) @WIN_UBANCHOR_OBJ_LINK@
TESTBOUND_SRC=testcode/testbound.c testcode/testpkts.c \
daemon/worker.c daemon/acl_list.c daemon/daemon.c daemon/stats.c \
@ -1175,7 +1175,7 @@ delayer.lo delayer.o: $(srcdir)/testcode/delayer.c config.h $(srcdir)/util/net_h
unbound-control.lo unbound-control.o: $(srcdir)/smallapp/unbound-control.c config.h \
$(srcdir)/util/log.h $(srcdir)/util/config_file.h $(srcdir)/util/locks.h $(srcdir)/util/net_help.h
unbound-anchor.lo unbound-anchor.o: $(srcdir)/smallapp/unbound-anchor.c config.h $(srcdir)/libunbound/unbound.h \
$(srcdir)/sldns/rrdef.h \
$(srcdir)/sldns/rrdef.h $(srcdir)/sldns/parseutil.h \
petal.lo petal.o: $(srcdir)/testcode/petal.c config.h \

4
external/unbound/config.h.cmake.in vendored
View File

@ -85,6 +85,10 @@
`SSL_COMP_get_compression_methods', and to 0 if you don't. */
#cmakedefine HAVE_DECL_SSL_COMP_GET_COMPRESSION_METHODS
/* Define to 1 if you have the declaration of `SSL_CTX_set_ecdh_auto', and to
0 if you don't. */
#cmakedefine HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
/* Define to 1 if you have the declaration of `strlcat', and to 0 if you
don't. */
#cmakedefine HAVE_DECL_STRLCAT

4
external/unbound/config.h.in vendored
View File

@ -82,6 +82,10 @@
`SSL_COMP_get_compression_methods', and to 0 if you don't. */
#undef HAVE_DECL_SSL_COMP_GET_COMPRESSION_METHODS
/* Define to 1 if you have the declaration of `SSL_CTX_set_ecdh_auto', and to
0 if you don't. */
#undef HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
/* Define to 1 if you have the declaration of `strlcat', and to 0 if you
don't. */
#undef HAVE_DECL_STRLCAT

58
external/unbound/configure vendored
View File

@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for unbound 1.5.4.
# Generated by GNU Autoconf 2.69 for unbound 1.5.5.
#
# Report bugs to <unbound-bugs@nlnetlabs.nl>.
#
@ -590,8 +590,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='unbound'
PACKAGE_TARNAME='unbound'
PACKAGE_VERSION='1.5.4'
PACKAGE_STRING='unbound 1.5.4'
PACKAGE_VERSION='1.5.5'
PACKAGE_STRING='unbound 1.5.5'
PACKAGE_BUGREPORT='unbound-bugs@nlnetlabs.nl'
PACKAGE_URL=''
@ -1389,7 +1389,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
\`configure' configures unbound 1.5.4 to adapt to many kinds of systems.
\`configure' configures unbound 1.5.5 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@ -1454,7 +1454,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
short | recursive ) echo "Configuration of unbound 1.5.4:";;
short | recursive ) echo "Configuration of unbound 1.5.5:";;
esac
cat <<\_ACEOF
@ -1629,7 +1629,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
unbound configure 1.5.4
unbound configure 1.5.5
generated by GNU Autoconf 2.69
Copyright (C) 2012 Free Software Foundation, Inc.
@ -2338,7 +2338,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
It was created by unbound $as_me 1.5.4, which was
It was created by unbound $as_me 1.5.5, which was
generated by GNU Autoconf 2.69. Invocation command line was
$ $0 $@
@ -2690,7 +2690,7 @@ UNBOUND_VERSION_MAJOR=1
UNBOUND_VERSION_MINOR=5
UNBOUND_VERSION_MICRO=4
UNBOUND_VERSION_MICRO=5
LIBUNBOUND_CURRENT=5
@ -16684,7 +16684,7 @@ rm -f core conftest.err conftest.$ac_objext \
conftest$ac_exeext conftest.$ac_ext
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for LibreSSL" >&5
$as_echo_n "checking for LibreSSL... " >&6; }
if grep OPENSSL_VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
if grep VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
$as_echo "yes" >&6; }
@ -16845,6 +16845,36 @@ fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL_SK_SSL_COMP_POP_FREE $ac_have_decl
_ACEOF
ac_fn_c_check_decl "$LINENO" "SSL_CTX_set_ecdh_auto" "ac_cv_have_decl_SSL_CTX_set_ecdh_auto" "
$ac_includes_default
#ifdef HAVE_OPENSSL_ERR_H
#include <openssl/err.h>
#endif
#ifdef HAVE_OPENSSL_RAND_H
#include <openssl/rand.h>
#endif
#ifdef HAVE_OPENSSL_CONF_H
#include <openssl/conf.h>
#endif
#ifdef HAVE_OPENSSL_ENGINE_H
#include <openssl/engine.h>
#endif
#include <openssl/ssl.h>
#include <openssl/evp.h>
"
if test "x$ac_cv_have_decl_SSL_CTX_set_ecdh_auto" = xyes; then :
ac_have_decl=1
else
ac_have_decl=0
fi
cat >>confdefs.h <<_ACEOF
#define HAVE_DECL_SSL_CTX_SET_ECDH_AUTO $ac_have_decl
_ACEOF
fi
@ -18150,6 +18180,8 @@ esac
fi
LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
ac_fn_c_check_func "$LINENO" "reallocarray" "ac_cv_func_reallocarray"
if test "x$ac_cv_func_reallocarray" = xyes; then :
$as_echo "#define HAVE_REALLOCARRAY 1" >>confdefs.h
@ -18164,8 +18196,6 @@ esac
fi
LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
if test "$USE_NSS" = "no"; then
ac_fn_c_check_func "$LINENO" "arc4random" "ac_cv_func_arc4random"
if test "x$ac_cv_func_arc4random" = xyes; then :
@ -18890,7 +18920,7 @@ _ACEOF
version=1.5.4
version=1.5.5
date=`date +'%b %e, %Y'`
@ -19405,7 +19435,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
This file was extended by unbound $as_me 1.5.4, which was
This file was extended by unbound $as_me 1.5.5, which was
generated by GNU Autoconf 2.69. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@ -19471,7 +19501,7 @@ _ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`"
ac_cs_version="\\
unbound config.status 1.5.4
unbound config.status 1.5.5
configured by $0, generated by GNU Autoconf 2.69,
with options \\"\$ac_cs_config\\"

9
external/unbound/configure.ac vendored
View File

@ -10,7 +10,7 @@ sinclude(dnstap/dnstap.m4)
# must be numbers. ac_defun because of later processing
m4_define([VERSION_MAJOR],[1])
m4_define([VERSION_MINOR],[5])
m4_define([VERSION_MICRO],[4])
m4_define([VERSION_MICRO],[5])
AC_INIT(unbound, m4_defn([VERSION_MAJOR]).m4_defn([VERSION_MINOR]).m4_defn([VERSION_MICRO]), unbound-bugs@nlnetlabs.nl, unbound)
AC_SUBST(UNBOUND_VERSION_MAJOR, [VERSION_MAJOR])
AC_SUBST(UNBOUND_VERSION_MINOR, [VERSION_MINOR])
@ -566,7 +566,7 @@ if test $USE_NSS = "no"; then
ACX_WITH_SSL
ACX_LIB_SSL
AC_MSG_CHECKING([for LibreSSL])
if grep OPENSSL_VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
if grep VERSION_TEXT $ssldir/include/openssl/opensslv.h | grep "LibreSSL" >/dev/null; then
AC_MSG_RESULT([yes])
AC_DEFINE([HAVE_LIBRESSL], [1], [Define if we have LibreSSL])
# libressl provides these compat functions, but they may also be
@ -578,7 +578,7 @@ fi
AC_CHECK_HEADERS([openssl/conf.h],,, [AC_INCLUDES_DEFAULT])
AC_CHECK_HEADERS([openssl/engine.h],,, [AC_INCLUDES_DEFAULT])
AC_CHECK_FUNCS([OPENSSL_config EVP_sha1 EVP_sha256 EVP_sha512 FIPS_mode])
AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free], [], [], [
AC_CHECK_DECLS([SSL_COMP_get_compression_methods,sk_SSL_COMP_pop_free,SSL_CTX_set_ecdh_auto], [], [], [
AC_INCLUDES_DEFAULT
#ifdef HAVE_OPENSSL_ERR_H
#include <openssl/err.h>
@ -998,9 +998,10 @@ AC_REPLACE_FUNCS(strlcat)
AC_REPLACE_FUNCS(strlcpy)
AC_REPLACE_FUNCS(memmove)
AC_REPLACE_FUNCS(gmtime_r)
AC_REPLACE_FUNCS(reallocarray)
dnl without CTIME, ARC4-functions and without reallocarray.
LIBOBJ_WITHOUT_CTIMEARC4="$LIBOBJS"
AC_SUBST(LIBOBJ_WITHOUT_CTIMEARC4)
AC_REPLACE_FUNCS(reallocarray)
if test "$USE_NSS" = "no"; then
AC_REPLACE_FUNCS(arc4random)
AC_REPLACE_FUNCS(arc4random_uniform)

8
external/unbound/daemon/daemon.c vendored
View File

@ -399,6 +399,12 @@ daemon_create_workers(struct daemon* daemon)
verbose(VERB_ALGO, "total of %d outgoing ports available", numport);
daemon->num = (daemon->cfg->num_threads?daemon->cfg->num_threads:1);
if(daemon->reuseport && (int)daemon->num < (int)daemon->num_ports) {
log_warn("cannot reduce num-threads to %d because so-reuseport "
"so continuing with %d threads.", (int)daemon->num,
(int)daemon->num_ports);
daemon->num = (int)daemon->num_ports;
}
daemon->workers = (struct worker**)calloc((size_t)daemon->num,
sizeof(struct worker*));
if(daemon->cfg->dnstap) {
@ -464,7 +470,7 @@ thread_start(void* arg)
#endif
#ifdef SO_REUSEPORT
if(worker->daemon->cfg->so_reuseport)
port_num = worker->thread_num;
port_num = worker->thread_num % worker->daemon->num_ports;
else
port_num = 0;
#endif

22
external/unbound/daemon/remote.c vendored
View File

@ -243,9 +243,9 @@ daemon_remote_create(struct config_file* cfg)
goto setup_error;
}
verbose(VERB_ALGO, "setup SSL certificates");
if (!SSL_CTX_use_certificate_file(rc->ctx,s_cert,SSL_FILETYPE_PEM)) {
if (!SSL_CTX_use_certificate_chain_file(rc->ctx,s_cert)) {
log_err("Error for server-cert-file: %s", s_cert);
log_crypto_err("Error in SSL_CTX use_certificate_file");
log_crypto_err("Error in SSL_CTX use_certificate_chain_file");
goto setup_error;
}
if(!SSL_CTX_use_PrivateKey_file(rc->ctx,s_key,SSL_FILETYPE_PEM)) {
@ -258,6 +258,23 @@ daemon_remote_create(struct config_file* cfg)
log_crypto_err("Error in SSL_CTX check_private_key");
goto setup_error;
}
#if HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
if(!SSL_CTX_set_ecdh_auto(rc->ctx,1)) {
log_crypto_err("Error in SSL_CTX_ecdh_auto, not enabling ECDHE");
}
#elif defined(USE_ECDSA)
if(1) {
EC_KEY *ecdh = EC_KEY_new_by_curve_name (NID_X9_62_prime256v1);
if (!ecdh) {
log_crypto_err("could not find p256, not enabling ECDHE");
} else {
if (1 != SSL_CTX_set_tmp_ecdh (rc->ctx, ecdh)) {
log_crypto_err("Error in SSL_CTX_set_tmp_ecdh, not enabling ECDHE");
}
EC_KEY_free (ecdh);
}
}
#endif
if(!SSL_CTX_load_verify_locations(rc->ctx, s_cert, NULL)) {
log_crypto_err("Error setting up SSL_CTX verify locations");
setup_error:
@ -1683,6 +1700,7 @@ parse_delegpt(SSL* ssl, char* args, uint8_t* nm, int allow_names)
}
}
}
dp->has_parent_side_NS = 1;
return dp;
}

2
external/unbound/daemon/worker.c vendored
View File

@ -568,7 +568,7 @@ answer_from_cache(struct worker* worker, struct query_info* qinfo,
if(rep->an_numrrsets > 0 && (rep->rrsets[0]->rk.type ==
htons(LDNS_RR_TYPE_CNAME) || rep->rrsets[0]->rk.type ==
htons(LDNS_RR_TYPE_DNAME))) {
if(!reply_check_cname_chain(rep)) {
if(!reply_check_cname_chain(qinfo, rep)) {
/* cname chain invalid, redo iterator steps */
verbose(VERB_ALGO, "Cache reply: cname chain broken");
bail_out:

70
external/unbound/doc/Changelog vendored
View File

@ -1,3 +1,73 @@
13 August 2015: Wouter
- 5011 implementation does not insist on all algorithms, when
harden-algo-downgrade is turned off.
- Reap the child process that libunbound spawns.
11 August 2015: Wouter
- Fix #694: configure script does not detect LibreSSL 2.2.2
4 August 2015: Wouter
- Document that local-zone nodefault matches exactly and transparent
can be used to release a subzone.
3 August 2015: Wouter
- Document in the manual more text about configuring locally served
zones.
- Fix 5011 anchor update timer after reload.
- Fix mktime in unbound-anchor not using UTC.
30 July 2015: Wouter
- please afl-gcc (llvm) for uninitialised variable warning.
- Added permit-small-holddown config to debug fast 5011 rollover.
24 July 2015: Wouter
- Fix #690: Reload fails when so-reuseport is yes after changing
num-threads.
- iana portlist update.
21 July 2015: Wouter
- Fix configure to detect SSL_CTX_set_ecdh_auto.
- iana portlist update.
20 July 2015: Wouter
- Enable ECDHE for servers. Where available, use
SSL_CTX_set_ecdh_auto() for TLS-wrapped server configurations to
enable ECDHE. Otherwise, manually offer curve p256.
Client connections should automatically use ECDHE when available.
(thanks Daniel Kahn Gillmor)
18 July 2015: Willem
- Allow certificate chain files to allow for intermediate certificates.
(thanks Daniel Kahn Gillmor)
13 July 2015: Wouter
- makedist produces sha1 and sha256 files for created binaries too.
9 July 2015: Wouter
- 1.5.4 release tag
- trunk has 1.5.5 in development.
- Fix #681: Setting forwarders with unbound-control forward
implicitly turns on forward-first.
29 June 2015: Wouter
- iana portlist update.
- Fix alloc with log for allocation size checks.
26 June 2015: Wouter
- Fix #677 Fix DNAME responses from cache that failed internal chain
test.
- iana portlist update.
22 June 2015: Wouter
- Fix #677 Fix CNAME corresponding to a DNAME was checked incorrectly
and was therefore always synthesized (thanks to Valentin Dietrich).
4 June 2015: Wouter
- RFC 7553 RR type URI support, is now enabled by default.
2 June 2015: Wouter
- Fix #674: Do not free pointers given by getenv.
29 May 2015: Wouter
- Fix that unparseable error responses are ratelimited.
- SOA negative TTL is capped at minimumttl in its rdata section.

5
external/unbound/doc/example.conf.in vendored
View File

@ -444,6 +444,9 @@ server:
# If the value 0 is given, missing anchors are not removed.
# keep-missing: 31622400 # 366 days
# debug option that allows very small holddown times for key rollover
# permit-small-holddown: no
# the amount of memory to use for the key cache.
# plain value in bytes or you can append k, m or G. default is "4Mb".
# key-cache-size: 4m
@ -623,6 +626,8 @@ remote-control:
# nameservers by hostname or by ipaddress. If you set stub-prime to yes,
# the list is treated as priming hints (default is no).
# With stub-first yes, it attempts without the stub if it fails.
# Consider adding domain-insecure: name and local-zone: name nodefault
# to the server: section if the stub is a locally served zone.
# stub-zone:
# name: "example.com"
# stub-addr: 192.0.2.68

13
external/unbound/doc/unbound.conf.5.in vendored
View File

@ -801,6 +801,10 @@ mechanism work with zones that perform regular (non\-5011) rollovers.
The default is 366 days. The value 0 does not remove missing anchors,
as per the RFC.
.TP
.B permit\-small\-holddown: \fI<yes or no>
Debug option that allows the autotrust 5011 rollover timers to assume
very small values. Default is no.
.TP
.B key\-cache\-size: \fI<number>
Number of bytes size of the key cache. Default is 4 megabytes.
A plain number is in bytes, append 'k', 'm' or 'g' for kilobytes, megabytes
@ -895,7 +899,8 @@ infected machines without answering the queries.
Used to turn off default contents for AS112 zones. The other types
also turn off default contents for the zone. The 'nodefault' option
has no other effect than turning off default contents for the
given zone.
given zone. Use \fInodefault\fR if you use exactly that zone, if you want to
use a subzone, use \fItransparent\fR.
.P
The default zones are localhost, reverse 127.0.0.1 and ::1, and the AS112
zones. The AS112 zones are reverse DNS zones for private use and reserved
@ -1124,6 +1129,12 @@ bit on replies for the private zone (authoritative servers do not set the
AD bit). This setup makes unbound capable of answering queries for the
private zone, and can even set the AD bit ('authentic'), but the AA
('authoritative') bit is not set on these replies.
.P
Consider adding \fBserver:\fR statements for \fBdomain\-insecure:\fR and
for \fBlocal\-zone:\fI name nodefault\fR for the zone if it is a locally
served zone. The insecure clause stops DNSSEC from invalidating the
zone. The local zone nodefault (or \fItransparent\fR) clause makes the
(reverse\-) zone bypass unbound's filtering of RFC1918 zones.
.TP
.B name: \fI<domain name>
Name of the stub zone.

2
external/unbound/iterator/iter_scrub.c vendored
View File

@ -372,7 +372,7 @@ scrub_normalize(sldns_buffer* pkt, struct msg_parse* msg,
/* check next cname */
uint8_t* t = NULL;
size_t tlen = 0;
if(!parse_get_cname_target(rrset, &t, &tlen))
if(!parse_get_cname_target(nx, &t, &tlen))
return 0;
if(dname_pkt_compare(pkt, alias, t) == 0) {
/* it's OK and better capitalized */

10
external/unbound/libunbound/libunbound.c vendored
View File

@ -65,6 +65,9 @@
#ifdef HAVE_PTHREAD
#include <signal.h>
#endif
#ifdef HAVE_SYS_WAIT_H
#include <sys/wait.h>
#endif
#if defined(UB_ON_WINDOWS) && defined (HAVE_WINDOWS_H)
#include <windows.h>
@ -218,6 +221,12 @@ static void ub_stop_bg(struct ub_ctx* ctx)
ub_thread_join(ctx->bg_tid);
} else {
lock_basic_unlock(&ctx->cfglock);
#ifndef UB_ON_WINDOWS
if(waitpid(ctx->bg_pid, NULL, 0) == -1) {
if(verbosity > 2)
log_err("waitpid: %s", strerror(errno));
}
#endif
}
}
else {
@ -1028,6 +1037,7 @@ ub_ctx_hosts(struct ub_ctx* ctx, const char* fname)
"\\hosts");
retval=ub_ctx_hosts(ctx, buf);
}
free(name);
return retval;
}
return UB_READFILE;

70
external/unbound/makedist.sh vendored
View File

@ -136,6 +136,43 @@ create_temp_dir () {
cd $temp_dir
}
# pass filename as $1 arg.
# creates file.sha1 and file.sha256
storehash () {
case $OSTYPE in
linux*)
sha=`sha1sum $1 | awk '{ print $1 }'`
sha256=`sha256sum $1 | awk '{ print $1 }'`
;;
freebsd*)
sha=`sha1 $1 | awk '{ print $5 }'`
sha256=`sha256 $1 | awk '{ print $5 }'`
;;
*)
# in case $OSTYPE is gone.
case `uname` in
Linux*)
sha=`sha1sum $1 | awk '{ print $1 }'`
sha256=`sha256sum $1 | awk '{ print $1 }'`
;;
FreeBSD*)
sha=`sha1 $1 | awk '{ print $5 }'`
sha256=`sha256 $1 | awk '{ print $5 }'`
;;
*)
sha=`sha1sum $1 | awk '{ print $1 }'`
sha256=`sha256sum $1 | awk '{ print $1 }'`
;;
esac
;;
esac
echo $sha > $1.sha1
echo $sha256 > $1.sha256
echo "hash of $1.{sha1,sha256}"
echo "sha1 $sha"
echo "sha256 $sha256"
}
SNAPSHOT="no"
RC="no"
@ -311,6 +348,8 @@ if [ "$DOWIN" = "yes" ]; then
mv unbound-$version.zip $cwd/.
cleanup
fi
storehash unbound_setup_$version.exe
storehash unbound-$version.zip
ls -lG unbound_setup_$version.exe
ls -lG unbound-$version.zip
info "Done"
@ -411,36 +450,7 @@ tar czf ../unbound-$version.tar.gz unbound-$version || error_cleanup "Failed to
cleanup
case $OSTYPE in
linux*)
sha=`sha1sum unbound-$version.tar.gz | awk '{ print $1 }'`
sha256=`sha256sum unbound-$version.tar.gz | awk '{ print $1 }'`
;;
freebsd*)
sha=`sha1 unbound-$version.tar.gz | awk '{ print $5 }'`
sha256=`sha256 unbound-$version.tar.gz | awk '{ print $5 }'`
;;
*)
# in case $OSTYPE is gone.
case `uname` in
Linux*)
sha=`sha1sum unbound-$version.tar.gz | awk '{ print $1 }'`
sha256=`sha256sum unbound-$version.tar.gz | awk '{ print $1 }'`
;;
FreeBSD*)
sha=`sha1 unbound-$version.tar.gz | awk '{ print $5 }'`
sha256=`sha256 unbound-$version.tar.gz | awk '{ print $5 }'`
;;
*)
sha=`sha1sum unbound-$version.tar.gz | awk '{ print $1 }'`
sha256=`sha256sum unbound-$version.tar.gz | awk '{ print $1 }'`
;;
esac
;;
esac
echo $sha > unbound-$version.tar.gz.sha1
echo $sha256 > unbound-$version.tar.gz.sha256
storehash unbound-$version.tar.gz
info "Unbound distribution created successfully."
info "SHA1sum: $sha"

2
external/unbound/services/cache/dns.c vendored
View File

@ -505,7 +505,7 @@ tomsg(struct module_env* env, struct query_info* q, struct reply_info* r,
return NULL;
if(r->an_numrrsets > 0 && (r->rrsets[0]->rk.type == htons(
LDNS_RR_TYPE_CNAME) || r->rrsets[0]->rk.type == htons(
LDNS_RR_TYPE_DNAME)) && !reply_check_cname_chain(r)) {
LDNS_RR_TYPE_DNAME)) && !reply_check_cname_chain(q, r)) {
/* cname chain is now invalid, reconstruct msg */
rrset_array_unlock(r->ref, r->rrset_count);
return NULL;

6
external/unbound/sldns/rrdef.c vendored
View File

@ -213,13 +213,11 @@ static const sldns_rdf_type type_eui48_wireformat[] = {
static const sldns_rdf_type type_eui64_wireformat[] = {
LDNS_RDF_TYPE_EUI64
};
#ifdef DRAFT_RRTYPES
static const sldns_rdf_type type_uri_wireformat[] = {
LDNS_RDF_TYPE_INT16,
LDNS_RDF_TYPE_INT16,
LDNS_RDF_TYPE_LONG_STR
};
#endif
static const sldns_rdf_type type_caa_wireformat[] = {
LDNS_RDF_TYPE_INT8,
LDNS_RDF_TYPE_TAG,
@ -590,12 +588,8 @@ static sldns_rr_descriptor rdata_field_descriptors[] = {
/* ANY: A request for all (available) records */
{LDNS_RR_TYPE_ANY, "ANY", 1, 1, type_0_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
#ifdef DRAFT_RRTYPES
/* 256 */
{LDNS_RR_TYPE_URI, "URI", 3, 3, type_uri_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
#else
{LDNS_RR_TYPE_NULL, "TYPE256", 1, 1, type_0_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },
#endif
/* 257 */
{LDNS_RR_TYPE_CAA, "CAA", 3, 3, type_caa_wireformat, LDNS_RDF_TYPE_NONE, LDNS_RR_NO_COMPRESS, 0 },

3
external/unbound/sldns/rrdef.h vendored
View File

@ -220,8 +220,7 @@ enum sldns_enum_rr_type
LDNS_RR_TYPE_MAILA = 254,
/** any type (wildcard) */
LDNS_RR_TYPE_ANY = 255,
/** draft-faltstrom-uri-06 */
LDNS_RR_TYPE_URI = 256,
LDNS_RR_TYPE_URI = 256, /* RFC 7553 */
LDNS_RR_TYPE_CAA = 257, /* RFC 6844 */
/** DNSSEC Trust Authorities */

3
external/unbound/smallapp/unbound-anchor.c vendored
View File

@ -117,6 +117,7 @@
#include "config.h"
#include "libunbound/unbound.h"
#include "sldns/rrdef.h"
#include "sldns/parseutil.h"
#include <expat.h>
#ifndef HAVE_EXPAT_H
#error "need libexpat to parse root-anchors.xml file."
@ -1328,7 +1329,7 @@ xml_convertdate(const char* str)
/* but ignore, (lenient) */
}
t = mktime(&tm);
t = sldns_mktime_from_utc(&tm);
if(t == (time_t)-1) {
if(verb) printf("xml_convertdate mktime failure\n");
return 0;

2
external/unbound/smallapp/unbound-control.c vendored
View File

@ -161,7 +161,7 @@ setup_ctx(struct config_file* cfg)
if(cfg->remote_control_use_cert) {
if(!(SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3) & SSL_OP_NO_SSLv3))
ssl_err("could not set SSL_OP_NO_SSLv3");
if(!SSL_CTX_use_certificate_file(ctx,c_cert,SSL_FILETYPE_PEM) ||
if(!SSL_CTX_use_certificate_chain_file(ctx,c_cert) ||
!SSL_CTX_use_PrivateKey_file(ctx,c_key,SSL_FILETYPE_PEM)
|| !SSL_CTX_check_private_key(ctx))
ssl_err("Error setting up SSL_CTX client key and cert");

18
external/unbound/testcode/petal.c vendored
View File

@ -236,12 +236,28 @@ setup_ctx(char* key, char* cert)
if(!ctx) print_exit("out of memory");
(void)SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2);
(void)SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3);
if(!SSL_CTX_use_certificate_file(ctx, cert, SSL_FILETYPE_PEM))
if(!SSL_CTX_use_certificate_chain_file(ctx, cert))
print_exit("cannot read cert");
if(!SSL_CTX_use_PrivateKey_file(ctx, key, SSL_FILETYPE_PEM))
print_exit("cannot read key");
if(!SSL_CTX_check_private_key(ctx))
print_exit("private key is not correct");
#if HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
if (!SSL_CTX_set_ecdh_auto(ctx,1))
if(verb>=1) printf("failed to set_ecdh_auto, not enabling ECDHE\n");
#elif defined(USE_ECDSA)
if(1) {
EC_KEY *ecdh = EC_KEY_new_by_curve_name (NID_X9_62_prime256v1);
if (!ecdh) {
if(verb>=1) printf("could not find p256, not enabling ECDHE\n");
} else {
if (1 != SSL_CTX_set_tmp_ecdh (ctx, ecdh)) {
if(verb>=1) printf("Error in SSL_CTX_set_tmp_ecdh, not enabling ECDHE\n");
}
EC_KEY_free(ecdh);
}
}
#endif
if(!SSL_CTX_load_verify_locations(ctx, cert, NULL))
print_exit("cannot load cert verify locations");
return ctx;

BIN
external/unbound/testdata/ctrl_pipe.tpkg vendored
View File

Binary file not shown.

161
external/unbound/testdata/fwd_any.rpl vendored
View File

@ -1,161 +0,0 @@
; This is a comment.
; config options go here.
forward-zone: name: "." forward-addr: 216.0.0.1
CONFIG_END
SCENARIO_BEGIN Test query and cache with type ANY
RANGE_BEGIN 0 1000
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
SECTION AUTHORITY
www.example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 10.20.30.50
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN ANY
SECTION ANSWER
;; different type in this answer.
www.example.com. IN TXT "text"
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN AAAA
SECTION ANSWER
www.example.com. IN AAAA ::5
ENTRY_END
ENTRY_BEGIN
MATCH opcode qname qtype
ADJUST copy_id
REPLY QR RD RA NOERROR
SECTION QUESTION
www.foo.com. IN ANY
SECTION ANSWER
www.foo.com. IN A 1.2.3.77
www.foo.com. IN AAAA ::77
ENTRY_END
RANGE_END
STEP 10 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; unneccesary nothing steps.
STEP 20 NOTHING
STEP 30 CHECK_ANSWER
ENTRY_BEGIN
MATCH opcode qname qtype
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
ENTRY_END
; test cache synthesis
STEP 40 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN ANY
ENTRY_END
STEP 50 NOTHING
STEP 60 CHECK_ANSWER
ENTRY_BEGIN
MATCH opcode qname qtype
SECTION QUESTION
www.example.com. IN ANY
SECTION ANSWER
www.example.com. IN A 10.20.30.40
ENTRY_END
; and again
; the synthesized result itself is not added to the cache
STEP 62 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN ANY
ENTRY_END
STEP 63 NOTHING
STEP 64 CHECK_ANSWER
ENTRY_BEGIN
MATCH opcode qname qtype
SECTION QUESTION
www.example.com. IN ANY
SECTION ANSWER
www.example.com. IN A 10.20.30.40
ENTRY_END
; AAAA lookup to add more data in cache
STEP 70 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN AAAA
ENTRY_END
STEP 80 NOTHING
STEP 90 CHECK_ANSWER
ENTRY_BEGIN
MATCH opcode qname qtype
SECTION QUESTION
www.example.com. IN AAAA
SECTION ANSWER
www.example.com. IN AAAA ::5
ENTRY_END
; test cache synthesis of AAAA, and two rrsets.
STEP 100 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN ANY
ENTRY_END
STEP 110 NOTHING
STEP 120 CHECK_ANSWER
ENTRY_BEGIN
MATCH opcode qname qtype
SECTION QUESTION
www.example.com. IN ANY
SECTION ANSWER
www.example.com. IN A 10.20.30.40
www.example.com. IN AAAA ::5
ENTRY_END
; test query that is not synthesized from cache.
STEP 130 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.foo.com. IN ANY
ENTRY_END
STEP 140 NOTHING
STEP 150 CHECK_ANSWER
ENTRY_BEGIN
MATCH opcode qname qtype
SECTION QUESTION
www.foo.com. IN ANY
SECTION ANSWER
www.foo.com. IN A 1.2.3.77
www.foo.com. IN AAAA ::77
ENTRY_END
SCENARIO_END

BIN
external/unbound/testdata/fwd_capsid_strip.tpkg vendored
View File

Binary file not shown.

BIN
external/unbound/testdata/fwd_capsid_white.tpkg vendored
View File

Binary file not shown.

273
external/unbound/testdata/iter_domain_sale.rpl vendored
View File

@ -1,273 +0,0 @@
; config options
server:
target-fetch-policy: "0 0 0 0 0"
stub-zone:
name: "."
stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET.
CONFIG_END
SCENARIO_BEGIN Test resolver with a domain sale
; and the old operator is nasty, keeps running his server with the old data.
; and lots of lookups keep going towards the domain.
; eventually, the NS record has to timeout.
; K.ROOT-SERVERS.NET.
RANGE_BEGIN 0 100
ADDRESS 193.0.14.129
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
. IN NS
SECTION ANSWER
. IN NS K.ROOT-SERVERS.NET.
SECTION ADDITIONAL
K.ROOT-SERVERS.NET. IN A 193.0.14.129
ENTRY_END
ENTRY_BEGIN
MATCH opcode subdomain
ADJUST copy_id copy_query
REPLY QR NOERROR
SECTION QUESTION
com. IN A
SECTION AUTHORITY
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
RANGE_END
; a.gtld-servers.net. (before sale of domain)
RANGE_BEGIN 0 20
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode subdomain
ADJUST copy_id copy_query
REPLY QR NOERROR
SECTION QUESTION
example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
RANGE_END
; a.gtld-servers.net. (after sale of domain)
RANGE_BEGIN 30 200
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode subdomain
ADJUST copy_id copy_query
REPLY QR NOERROR
SECTION QUESTION
example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 8.8.8.8
ENTRY_END
RANGE_END
; ns.example.com. first owner
RANGE_BEGIN 0 200
ADDRESS 1.2.3.4
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
ENTRY_BEGIN
MATCH opcode qname
ADJUST copy_id copy_query
REPLY QR AA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ENTRY_END
; nxdomains for any name,type
; last in RANGE so that it matches everything left over.
; it includes the NS record.
ENTRY_BEGIN
MATCH opcode
ADJUST copy_id copy_query
REPLY QR AA NXDOMAIN
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
SECTION AUTHORITY
example.com. 3600 IN SOA a. b. 1 2 3 4 5
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ENTRY_END
RANGE_END
; ns.example.com. new owner
RANGE_BEGIN 0 200
ADDRESS 8.8.8.8
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 8.8.8.8
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 88.88.88.88
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 8.8.8.8
ENTRY_END
RANGE_END
; Fetch the old record from the old owner.
STEP 1 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 5 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ENTRY_END
; the domain is sold (right at this time).
; but the information stays in the cache.
; after 1800 secs still the cached answer
STEP 20 TIME_PASSES ELAPSE 1800
STEP 30 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 40 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 1800 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 1800 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 1800 IN A 1.2.3.4
ENTRY_END
; and ask another query
STEP 50 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
nx1.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 60 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NXDOMAIN
SECTION QUESTION
nx1.example.com. IN A
SECTION ANSWER
SECTION AUTHORITY
; at TTL 5 because TTL is capped at min-ttl of 5 in rdata of SOA
example.com. 5 IN SOA a. b. 1 2 3 4 5
example.com. 1800 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 1800 IN A 1.2.3.4
ENTRY_END
; after another 1900 seconds the domain must have timed out.
STEP 70 TIME_PASSES ELAPSE 1900
; the NS record should have timed out.
STEP 80 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 90 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 88.88.88.88
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 8.8.8.8
ENTRY_END
SCENARIO_END

342
external/unbound/testdata/iter_domain_sale_nschange.rpl vendored
View File

@ -1,342 +0,0 @@
; config options
server:
target-fetch-policy: "0 0 0 0 0"
stub-zone:
name: "."
stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET.
CONFIG_END
SCENARIO_BEGIN Test resolver with a domain sale and NS changes
; and the old operator is nasty, keeps running his server with the old data.
; and lots of lookups keep going towards the domain.
; and the old server is changing the NS record of the old domain.
; K.ROOT-SERVERS.NET.
RANGE_BEGIN 0 100
ADDRESS 193.0.14.129
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
. IN NS
SECTION ANSWER
. IN NS K.ROOT-SERVERS.NET.
SECTION ADDITIONAL
K.ROOT-SERVERS.NET. IN A 193.0.14.129
ENTRY_END
ENTRY_BEGIN
MATCH opcode subdomain
ADJUST copy_id copy_query
REPLY QR NOERROR
SECTION QUESTION
com. IN A
SECTION AUTHORITY
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
RANGE_END
; a.gtld-servers.net. (before sale of domain)
RANGE_BEGIN 0 20
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode subdomain
ADJUST copy_id copy_query
REPLY QR NOERROR
SECTION QUESTION
example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
RANGE_END
; a.gtld-servers.net. (after sale of domain)
RANGE_BEGIN 30 200
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode subdomain
ADJUST copy_id copy_query
REPLY QR NOERROR
SECTION QUESTION
example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 8.8.8.8
ENTRY_END
RANGE_END
; ns.example.com. first owner
RANGE_BEGIN 0 30
ADDRESS 1.2.3.4
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
ENTRY_BEGIN
MATCH opcode qname
ADJUST copy_id copy_query
REPLY QR AA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ENTRY_END
; nxdomains for any name,type
; last in RANGE so that it matches everything left over.
; it includes the NS record.
ENTRY_BEGIN
MATCH opcode
ADJUST copy_id copy_query
REPLY QR AA NXDOMAIN
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
SECTION AUTHORITY
example.com. 3600 IN SOA a. b. 1 2 3 4 5
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ENTRY_END
RANGE_END
; ns.example.com. first owner, NS changed
RANGE_BEGIN 40 200
ADDRESS 1.2.3.4
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS nsb.example.com.
SECTION ADDITIONAL
nsb.example.com. IN A 1.2.3.4
ENTRY_END
ENTRY_BEGIN
MATCH opcode qname
ADJUST copy_id copy_query
REPLY QR AA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 3600 IN NS nsb.example.com.
SECTION ADDITIONAL
nsb.example.com. 3600 IN A 1.2.3.4
ENTRY_END
; nxdomains for any name,type
; last in RANGE so that it matches everything left over.
; it includes the NS record.
ENTRY_BEGIN
MATCH opcode
ADJUST copy_id copy_query
REPLY QR AA NXDOMAIN
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
SECTION AUTHORITY
example.com. 3600 IN SOA a. b. 1 2 3 4 5
example.com. 3600 IN NS nsb.example.com.
SECTION ADDITIONAL
nsb.example.com. 3600 IN A 1.2.3.4
ENTRY_END
RANGE_END
; ns.example.com. new owner
RANGE_BEGIN 0 200
ADDRESS 8.8.8.8
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 8.8.8.8
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 88.88.88.88
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 8.8.8.8
ENTRY_END
RANGE_END
; Fetch the old record from the old owner.
STEP 1 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 5 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ENTRY_END
; the domain is sold (right at this time).
; but the information stays in the cache.
; after 1800 secs still the cached answer
STEP 20 TIME_PASSES ELAPSE 1800
STEP 30 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 40 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 1800 IN A 10.20.30.40
SECTION AUTHORITY
example.com. 1800 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 1800 IN A 1.2.3.4
ENTRY_END
; and ask another query
STEP 50 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
nx1.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 60 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NXDOMAIN
SECTION QUESTION
nx1.example.com. IN A
SECTION ANSWER
SECTION AUTHORITY
; at TTL 5 because TTL capped at ttl of minttl in rdata of SOA.
example.com. 5 IN SOA a. b. 1 2 3 4 5
example.com. 3600 IN NS nsb.example.com.
SECTION ADDITIONAL
nsb.example.com. 3600 IN A 1.2.3.4
ENTRY_END
STEP 62 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
nx1.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 63 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NXDOMAIN
SECTION QUESTION
nx1.example.com. IN A
SECTION ANSWER
SECTION AUTHORITY
; at TTL 5 because TTL capped at ttl of minttl in rdata of SOA.
example.com. 5 IN SOA a. b. 1 2 3 4 5
example.com. 1800 IN NS nsb.example.com.
SECTION ADDITIONAL
nsb.example.com. 3600 IN A 1.2.3.4
ENTRY_END
; after another 1900 seconds the domain must have timed out.
STEP 70 TIME_PASSES ELAPSE 1900
; the NS record should have timed out.
STEP 80 QUERY
ENTRY_BEGIN
REPLY RD
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 90 CHECK_ANSWER
ENTRY_BEGIN
MATCH all ttl
REPLY QR RD RA NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 88.88.88.88
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. 3600 IN A 8.8.8.8
ENTRY_END
SCENARIO_END

BIN
external/unbound/testdata/root_anchor.tpkg vendored
View File

Binary file not shown.

151
external/unbound/testdata/val_spurious_ns.rpl vendored
View File

@ -1,151 +0,0 @@
; config options
; The island of trust is at example.com
server:
trust-anchor: "example.com. 3600 IN DS 2854 3 1 46e4ffc6e9a4793b488954bd3f0cc6af0dfb201b"
val-override-date: "20070916134226"
target-fetch-policy: "0 0 0 0 0"
stub-zone:
name: "."
stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET.
CONFIG_END
SCENARIO_BEGIN Test validator with spurious unsigned NS in auth section
; K.ROOT-SERVERS.NET.
RANGE_BEGIN 0 100
ADDRESS 193.0.14.129
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
. IN NS
SECTION ANSWER
. IN NS K.ROOT-SERVERS.NET.
SECTION ADDITIONAL
K.ROOT-SERVERS.NET. IN A 193.0.14.129
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION AUTHORITY
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
RANGE_END
; a.gtld-servers.net.
RANGE_BEGIN 0 100
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
RANGE_END
; ns.example.com.
RANGE_BEGIN 0 100
ADDRESS 1.2.3.4
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ENTRY_END
; response to DNSKEY priming query
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
example.com. IN DNSKEY
SECTION ANSWER
example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJII s70j+sDS/UT2QRp61SE7S3E EXopNXoFE73JLRmvpi/UrOO/Vz4Se 6wXv/CYCKjGw06U4WRgR YXcpEhJROyNapmdIKSx hOzfLVE1gqA0PweZR8d tY3aNQSRn3sPpwJr6Mi /PqQKAMMrZ9ckJpf1+b QMOOvxgzz2U1GS18b3y ZKcgTMEaJzd/GZYzi/B N2DzQ0MsrSwYXfsNLFO Bbs8PJMW4LYIxeeOe6rUgkWOF 7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}
example.com. 3600 IN RRSIG DNSKEY 3 2 3600 20070926134802 20070829134802 2854 example.com. MCwCFG1yhRNtTEa3Eno2zhVVuy2EJX3wAhQeLyUp6+UXcpC5qGNu9tkrTEgPUg== ;{id = 2854}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ENTRY_END
; response to query of interest
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
;example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
ENTRY_END
RANGE_END
STEP 1 QUERY
ENTRY_BEGIN
REPLY RD DO
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 10 CHECK_ANSWER
ENTRY_BEGIN
MATCH all
REPLY QR RD RA AD DO NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
SECTION AUTHORITY
; removed by spurious NS record removal code
;;example.com. IN NS ns.example.com.
;;example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
ENTRY_END
SCENARIO_END

182
external/unbound/testdata/val_ta_algo_dnskey_dp.rpl vendored
View File

@ -1,182 +0,0 @@
; config options
; The island of trust is at example.com
server:
trust-anchor: "example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}"
trust-anchor: "example.com. 3600 IN DS 30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512"
val-override-date: "20070916134226"
target-fetch-policy: "0 0 0 0 0"
harden-algo-downgrade: no
stub-zone:
name: "."
stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET.
CONFIG_END
SCENARIO_BEGIN Test validator with multiple algorithm trust anchor without harden
; K.ROOT-SERVERS.NET.
RANGE_BEGIN 0 100
ADDRESS 193.0.14.129
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
. IN NS
SECTION ANSWER
. IN NS K.ROOT-SERVERS.NET.
SECTION ADDITIONAL
K.ROOT-SERVERS.NET. IN A 193.0.14.129
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION AUTHORITY
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
RANGE_END
; a.gtld-servers.net.
RANGE_BEGIN 0 100
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
RANGE_END
; ns.example.com.
RANGE_BEGIN 0 100
ADDRESS 1.2.3.4
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
ns.example.com. IN AAAA
SECTION ANSWER
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
; response to DNSKEY priming query
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
example.com. IN DNSKEY
SECTION ANSWER
example.com. 3600 IN DNSKEY 256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}
example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 512b}
example.com. 3600 IN RRSIG DNSKEY 3 2 3600 20070926134150 20070829134150 2854 example.com. AKIIYDOGHogglFqJK94ZtOnF7EfGikgAyloMNRSMCrQgFaFkmcOyjrc= ;{id = 2854}
example.com. 3600 IN RRSIG DNSKEY 5 2 3600 20070926134150 20070829134150 30899 example.com. J55fsz1GGMnngc4r50xvXDUdaVMlfcLKLVsfMhwNLF+ERac5XV/lLRAc/aSER+qQdsSo0CrjYjy1wat7YQpDAA== ;{id = 30899}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
; response to query of interest
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
RANGE_END
STEP 1 QUERY
ENTRY_BEGIN
REPLY RD DO
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 10 CHECK_ANSWER
ENTRY_BEGIN
MATCH all
REPLY QR RD RA AD DO NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
SCENARIO_END

185
external/unbound/testdata/val_ta_algo_missing_dp.rpl vendored
View File

@ -1,185 +0,0 @@
; config options
; The island of trust is at example.com
server:
trust-anchor: "example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 1688b}"
trust-anchor: "example.com. 3600 IN DS 30899 5 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512"
trust-anchor: "example.com. 3600 IN DS 30899 7 1 d4bf9d2e10f6d76840d42ef5913022abcd0bf512"
val-override-date: "20070916134226"
target-fetch-policy: "0 0 0 0 0"
harden-algo-downgrade: no
stub-zone:
name: "."
stub-addr: 193.0.14.129 # K.ROOT-SERVERS.NET.
CONFIG_END
SCENARIO_BEGIN Test validator with multiple algorithm missing one
; K.ROOT-SERVERS.NET.
RANGE_BEGIN 0 100
ADDRESS 193.0.14.129
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
. IN NS
SECTION ANSWER
. IN NS K.ROOT-SERVERS.NET.
SECTION ADDITIONAL
K.ROOT-SERVERS.NET. IN A 193.0.14.129
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION AUTHORITY
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
RANGE_END
; a.gtld-servers.net.
RANGE_BEGIN 0 100
ADDRESS 192.5.6.30
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
com. IN NS
SECTION ANSWER
com. IN NS a.gtld-servers.net.
SECTION ADDITIONAL
a.gtld-servers.net. IN A 192.5.6.30
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION AUTHORITY
example.com. IN NS ns.example.com.
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ENTRY_END
RANGE_END
; ns.example.com.
RANGE_BEGIN 0 100
ADDRESS 1.2.3.4
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
example.com. IN NS
SECTION ANSWER
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR AA NOERROR
SECTION QUESTION
ns.example.com. IN AAAA
SECTION ANSWER
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
; response to DNSKEY priming query
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
example.com. IN DNSKEY
SECTION ANSWER
example.com. 3600 IN DNSKEY 256 3 5 AQPQ41chR9DEHt/aIzIFAqanbDlRflJoRs5yz1jFsoRIT7dWf0r+PeDuewdxkszNH6wnU4QL8pfKFRh5PIYVBLK3 ;{id = 30899 (zsk), size = 512b}
example.com. 3600 IN DNSKEY 256 3 3 ALXLUsWqUrY3JYER3T4TBJIIs70j+sDS/UT2QRp61SE7S3EEXopNXoFE73JLRmvpi/UrOO/Vz4Se6wXv/CYCKjGw06U4WRgRYXcpEhJROyNapmdIKSxhOzfLVE1gqA0PweZR8dtY3aNQSRn3sPpwJr6Mi/PqQKAMMrZ9ckJpf1+bQMOOvxgzz2U1GS18b3yZKcgTMEaJzd/GZYzi/BN2DzQ0MsrSwYXfsNLFOBbs8PJMW4LYIxeeOe6rUgkWOF7CC9Dh/dduQ1QrsJhmZAEFfd6ByYV+ ;{id = 2854 (zsk), size = 512b}
example.com. 3600 IN RRSIG DNSKEY 3 2 3600 20070926134150 20070829134150 2854 example.com. AKIIYDOGHogglFqJK94ZtOnF7EfGikgAyloMNRSMCrQgFaFkmcOyjrc= ;{id = 2854}
example.com. 3600 IN RRSIG DNSKEY 5 2 3600 20070926134150 20070829134150 30899 example.com. J55fsz1GGMnngc4r50xvXDUdaVMlfcLKLVsfMhwNLF+ERac5XV/lLRAc/aSER+qQdsSo0CrjYjy1wat7YQpDAA== ;{id = 30899}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926135752 20070829135752 2854 example.com. MC0CFQCMSWxVehgOQLoYclB9PIAbNP229AIUeH0vNNGJhjnZiqgIOKvs1EhzqAo= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
; response to query of interest
ENTRY_BEGIN
MATCH opcode qtype qname
ADJUST copy_id
REPLY QR NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. IN A 10.20.30.40
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
SECTION AUTHORITY
example.com. IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. IN A 1.2.3.4
www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
RANGE_END
STEP 1 QUERY
ENTRY_BEGIN
REPLY RD DO
SECTION QUESTION
www.example.com. IN A
ENTRY_END
; recursion happens here.
STEP 10 CHECK_ANSWER
ENTRY_BEGIN
MATCH all
REPLY QR RD RA AD DO NOERROR
SECTION QUESTION
www.example.com. IN A
SECTION ANSWER
www.example.com. 3600 IN A 10.20.30.40
www.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. JNWECShNE+nCLQwOXJJ3xpUkh2G+FCh5nk8uYAHIVQRse/BIvCMSlvRrtVyw9RnXvk5RR2bEgN0pRdLWW7ug5Q== ;{id = 30899}
www.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFC99iE9K5y2WNgI0gFvBWaTi9wm6AhUAoUqOpDtG5Zct+Qr9F3mSdnbc6V4= ;{id = 2854}
SECTION AUTHORITY
example.com. 3600 IN NS ns.example.com.
example.com. 3600 IN RRSIG NS 3 2 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCN+qHdJxoI/2tNKwsb08pra/G7aAIUAWA5sDdJTbrXA1/3OaesGBAO3sI= ;{id = 2854}
example.com. 3600 IN RRSIG NS 5 2 3600 20070926134150 20070829134150 30899 example.com. YTqtYba73HIOQuPr5oDyIX9pfmz1ybEBjwlD/jUgcPmFINUOZ9FeqG6ywgRKwn4AizkKTK00p1sxZYMKxl91wg== ;{id = 30899}
SECTION ADDITIONAL
ns.example.com. 3600 IN A 1.2.3.4
ns.example.com. 3600 IN RRSIG A 3 3 3600 20070926134150 20070829134150 2854 example.com. MC0CFQCQMyTjn7WWwpwAR1LlVeLpRgZGuQIUCcJDEkwAuzytTDRlYK7nIMwH1CM= ;{id = 2854}
ns.example.com. 3600 IN RRSIG A 5 3 3600 20070926134150 20070829134150 30899 example.com. Dn1ziMKrc3NdJkSv8g61Y9WNk3+BAuwCwnYzAZiHmkejkSCPViLJN7+f4Conp9l8LkTl50ZnLgoYrrUYNhMj6w== ;{id = 30899}
ENTRY_END
SCENARIO_END

3
external/unbound/util/alloc.c vendored
View File

@ -364,6 +364,9 @@ void *unbound_stat_malloc(size_t size)
#ifdef calloc
#undef calloc
#endif
#ifndef INT_MAX
#define INT_MAX (((int)-1)>>1)
#endif
/** calloc with stats */
void *unbound_stat_calloc(size_t nmemb, size_t size)
{

8
external/unbound/util/config_file.c vendored
View File

@ -70,6 +70,8 @@
uid_t cfg_uid = (uid_t)-1;
/** from cfg username, after daemonise setup performed */
gid_t cfg_gid = (gid_t)-1;
/** for debug allow small timeout values for fast rollovers */
int autr_permit_small_holddown = 0;
/** global config during parsing */
struct config_parser_state* cfg_parser = 0;
@ -200,6 +202,7 @@ config_create(void)
cfg->add_holddown = 30*24*3600;
cfg->del_holddown = 30*24*3600;
cfg->keep_missing = 366*24*3600; /* one year plus a little leeway */
cfg->permit_small_holddown = 0;
cfg->key_cache_size = 4 * 1024 * 1024;
cfg->key_cache_slabs = 4;
cfg->neg_cache_size = 1 * 1024 * 1024;
@ -444,6 +447,9 @@ int config_set_option(struct config_file* cfg, const char* opt,
else S_UNSIGNED_OR_ZERO("add-holddown:", add_holddown)
else S_UNSIGNED_OR_ZERO("del-holddown:", del_holddown)
else S_UNSIGNED_OR_ZERO("keep-missing:", keep_missing)
else if(strcmp(opt, "permit-small-holddown:") == 0)
{ IS_YES_OR_NO; cfg->permit_small_holddown = (strcmp(val, "yes") == 0);
autr_permit_small_holddown = cfg->permit_small_holddown; }
else S_MEMSIZE("key-cache-size:", key_cache_size)
else S_POW2("key-cache-slabs:", key_cache_slabs)
else S_MEMSIZE("neg-cache-size:", neg_cache_size)
@ -705,6 +711,7 @@ config_get_option(struct config_file* cfg, const char* opt,
else O_UNS(opt, "add-holddown", add_holddown)
else O_UNS(opt, "del-holddown", del_holddown)
else O_UNS(opt, "keep-missing", keep_missing)
else O_YNO(opt, "permit-small-holddown", permit_small_holddown)
else O_MEM(opt, "key-cache-size", key_cache_size)
else O_DEC(opt, "key-cache-slabs", key_cache_slabs)
else O_MEM(opt, "neg-cache-size", neg_cache_size)
@ -1243,6 +1250,7 @@ config_apply(struct config_file* config)
MINIMAL_RESPONSES = config->minimal_responses;
RRSET_ROUNDROBIN = config->rrset_roundrobin;
log_set_time_asc(config->log_time_ascii);
autr_permit_small_holddown = config->permit_small_holddown;
}
void config_lookup_uid(struct config_file* cfg)

4
external/unbound/util/config_file.h vendored
View File

@ -269,6 +269,8 @@ struct config_file {
unsigned int del_holddown;
/** autotrust keep_missing time, in seconds. 0 is forever. */
unsigned int keep_missing;
/** permit small holddown values, allowing 5011 rollover very fast */
int permit_small_holddown;
/** size of the key cache */
size_t key_cache_size;
@ -368,6 +370,8 @@ struct config_file {
extern uid_t cfg_uid;
/** from cfg username, after daemonise setup performed */
extern gid_t cfg_gid;
/** debug and enable small timeouts */
extern int autr_permit_small_holddown;
/**
* Stub config options

2656
external/unbound/util/configlexer.c vendored
View File

File diff suppressed because it is too large Load Diff

1
external/unbound/util/configlexer.lex vendored
View File

@ -306,6 +306,7 @@ val-nsec3-keysize-iterations{COLON} {
add-holddown{COLON} { YDVAR(1, VAR_ADD_HOLDDOWN) }
del-holddown{COLON} { YDVAR(1, VAR_DEL_HOLDDOWN) }
keep-missing{COLON} { YDVAR(1, VAR_KEEP_MISSING) }
permit-small-holddown{COLON} { YDVAR(1, VAR_PERMIT_SMALL_HOLDDOWN) }
use-syslog{COLON} { YDVAR(1, VAR_USE_SYSLOG) }
log-time-ascii{COLON} { YDVAR(1, VAR_LOG_TIME_ASCII) }
log-queries{COLON} { YDVAR(1, VAR_LOG_QUERIES) }

1214
external/unbound/util/configparser.c vendored
View File

File diff suppressed because it is too large Load Diff

6
external/unbound/util/configparser.h vendored
View File

@ -203,7 +203,8 @@ extern int yydebug;
VAR_RATELIMIT_BELOW_DOMAIN = 412,
VAR_RATELIMIT_FACTOR = 413,
VAR_CAPS_WHITELIST = 414,
VAR_CACHE_MAX_NEGATIVE_TTL = 415
VAR_CACHE_MAX_NEGATIVE_TTL = 415,
VAR_PERMIT_SMALL_HOLDDOWN = 416
};
#endif
/* Tokens. */
@ -365,6 +366,7 @@ extern int yydebug;
#define VAR_RATELIMIT_FACTOR 413
#define VAR_CAPS_WHITELIST 414
#define VAR_CACHE_MAX_NEGATIVE_TTL 415
#define VAR_PERMIT_SMALL_HOLDDOWN 416
@ -378,7 +380,7 @@ typedef union YYSTYPE
/* Line 2058 of yacc.c */
#line 382 "util/configparser.h"
#line 384 "util/configparser.h"
} YYSTYPE;
# define YYSTYPE_IS_TRIVIAL 1
# define yystype YYSTYPE /* obsolescent; will be withdrawn */

14
external/unbound/util/configparser.y vendored
View File

@ -121,7 +121,7 @@ extern struct config_parser_state* cfg_parser;
%token VAR_HARDEN_ALGO_DOWNGRADE VAR_IP_TRANSPARENT
%token VAR_RATELIMIT VAR_RATELIMIT_SLABS VAR_RATELIMIT_SIZE
%token VAR_RATELIMIT_FOR_DOMAIN VAR_RATELIMIT_BELOW_DOMAIN VAR_RATELIMIT_FACTOR
%token VAR_CAPS_WHITELIST VAR_CACHE_MAX_NEGATIVE_TTL
%token VAR_CAPS_WHITELIST VAR_CACHE_MAX_NEGATIVE_TTL VAR_PERMIT_SMALL_HOLDDOWN
%%
toplevelvars: /* empty */ | toplevelvars toplevelvar ;
@ -185,7 +185,8 @@ content_server: server_num_threads | server_verbosity | server_port |
server_ip_transparent | server_ratelimit | server_ratelimit_slabs |
server_ratelimit_size | server_ratelimit_for_domain |
server_ratelimit_below_domain | server_ratelimit_factor |
server_caps_whitelist | server_cache_max_negative_ttl
server_caps_whitelist | server_cache_max_negative_ttl |
server_permit_small_holddown
;
stubstart: VAR_STUB_ZONE
{
@ -1125,6 +1126,15 @@ server_keep_missing: VAR_KEEP_MISSING STRING_ARG
free($2);
}
;
server_permit_small_holddown: VAR_PERMIT_SMALL_HOLDDOWN STRING_ARG
{
OUTYY(("P(server_permit_small_holddown:%s)\n", $2));
if(strcmp($2, "yes") != 0 && strcmp($2, "no") != 0)
yyerror("expected yes or no.");
else cfg_parser->cfg->permit_small_holddown =
(strcmp($2, "yes")==0);
free($2);
}
server_key_cache_size: VAR_KEY_CACHE_SIZE STRING_ARG
{
OUTYY(("P(server_key_cache_size:%s)\n", $2));

2
external/unbound/util/data/msgencode.c vendored
View File

@ -283,7 +283,7 @@ compress_owner(struct ub_packed_rrset_key* key, sldns_buffer* pkt,
size_t owner_pos, uint16_t* owner_ptr, int owner_labs)
{
struct compress_tree_node* p;
struct compress_tree_node** insertpt;
struct compress_tree_node** insertpt = NULL;
if(!*owner_ptr) {
/* compress first time dname */
if((p = compress_tree_lookup(tree, key->rk.dname,

6
external/unbound/util/data/msgreply.c vendored
View File

@ -822,13 +822,13 @@ log_query_info(enum verbosity_value v, const char* str,
}
int
reply_check_cname_chain(struct reply_info* rep)
reply_check_cname_chain(struct query_info* qinfo, struct reply_info* rep)
{
/* check only answer section rrs for matching cname chain.
* the cache may return changed rdata, but owner names are untouched.*/
size_t i;
uint8_t* sname = rep->rrsets[0]->rk.dname;
size_t snamelen = rep->rrsets[0]->rk.dname_len;
uint8_t* sname = qinfo->qname;
size_t snamelen = qinfo->qname_len;
for(i=0; i<rep->an_numrrsets; i++) {
uint16_t t = ntohs(rep->rrsets[i]->rk.type);
if(t == LDNS_RR_TYPE_DNAME)

3
external/unbound/util/data/msgreply.h vendored
View File

@ -359,10 +359,11 @@ uint8_t* reply_find_final_cname_target(struct query_info* qinfo,
/**
* Check if cname chain in cached reply is still valid.
* @param qinfo: query info with query name.
* @param rep: reply to check.
* @return: true if valid, false if invalid.
*/
int reply_check_cname_chain(struct reply_info* rep);
int reply_check_cname_chain(struct query_info* qinfo, struct reply_info* rep);
/**
* Check security status of all RRs in the message.

9
external/unbound/util/iana_ports.inc vendored
View File

@ -1066,7 +1066,6 @@
1404,
1405,
1406,
1407,
1408,
1409,
1410,
@ -4667,6 +4666,7 @@
7725,
7726,
7727,
7728,
7734,
7738,
7741,
@ -4781,6 +4781,7 @@
8301,
8320,
8321,
8322,
8351,
8376,
8377,
@ -4788,6 +4789,7 @@
8379,
8380,
8383,
8384,
8400,
8401,
8402,
@ -4804,6 +4806,7 @@
8474,
8500,
8501,
8503,
8554,
8555,
8567,
@ -5034,6 +5037,7 @@
10200,
10201,
10252,
10253,
10260,
10288,
10439,
@ -5168,6 +5172,8 @@
17220,
17221,
17222,
17224,
17225,
17234,
17235,
17500,
@ -5380,6 +5386,7 @@
40843,
40853,
41111,
41230,
41794,
41795,
42508,

23
external/unbound/util/net_help.c vendored
View File

@ -629,9 +629,9 @@ void* listen_sslctx_create(char* key, char* pem, char* verifypem)
SSL_CTX_free(ctx);
return NULL;
}
if(!SSL_CTX_use_certificate_file(ctx, pem, SSL_FILETYPE_PEM)) {
if(!SSL_CTX_use_certificate_chain_file(ctx, pem)) {
log_err("error for cert file: %s", pem);
log_crypto_err("error in SSL_CTX use_certificate_file");
log_crypto_err("error in SSL_CTX use_certificate_chain_file");
SSL_CTX_free(ctx);
return NULL;
}
@ -647,6 +647,23 @@ void* listen_sslctx_create(char* key, char* pem, char* verifypem)
SSL_CTX_free(ctx);
return NULL;
}
#if HAVE_DECL_SSL_CTX_SET_ECDH_AUTO
if(!SSL_CTX_set_ecdh_auto(ctx,1)) {
log_crypto_err("Error in SSL_CTX_ecdh_auto, not enabling ECDHE");
}
#elif defined(USE_ECDSA)
if(1) {
EC_KEY *ecdh = EC_KEY_new_by_curve_name (NID_X9_62_prime256v1);
if (!ecdh) {
log_crypto_err("could not find p256, not enabling ECDHE");
} else {
if (1 != SSL_CTX_set_tmp_ecdh (ctx, ecdh)) {
log_crypto_err("Error in SSL_CTX_set_tmp_ecdh, not enabling ECDHE");
}
EC_KEY_free (ecdh);
}
}
#endif
if(verifypem && verifypem[0]) {
if(!SSL_CTX_load_verify_locations(ctx, verifypem, NULL)) {
@ -684,7 +701,7 @@ void* connect_sslctx_create(char* key, char* pem, char* verifypem)
return NULL;
}
if(key && key[0]) {
if(!SSL_CTX_use_certificate_file(ctx, pem, SSL_FILETYPE_PEM)) {
if(!SSL_CTX_use_certificate_chain_file(ctx, pem)) {
log_err("error in client certificate %s", pem);
log_crypto_err("error in certificate file");
SSL_CTX_free(ctx);

31
external/unbound/validator/autotrust.c vendored
View File

@ -1225,7 +1225,7 @@ verify_dnskey(struct module_env* env, struct val_env* ve,
{
char* reason = NULL;
uint8_t sigalg[ALGO_NEEDS_MAX+1];
int downprot = 1;
int downprot = env->cfg->harden_algo_downgrade;
enum sec_status sec = val_verify_DNSKEY_with_TA(env, ve, rrset,
tp->ds_rrset, tp->dnskey_rrset, downprot?sigalg:NULL, &reason);
/* sigalg is ignored, it returns algorithms signalled to exist, but
@ -1447,9 +1447,11 @@ set_tp_times(struct trust_anchor* tp, time_t rrsig_exp_interval,
if(rrsig_exp_interval/2 < x)
x = rrsig_exp_interval/2;
/* MAX(1hr, x) */
if(x < 3600)
tp->autr->query_interval = 3600;
else tp->autr->query_interval = x;
if(!autr_permit_small_holddown) {
if(x < 3600)
tp->autr->query_interval = 3600;
else tp->autr->query_interval = x;
} else tp->autr->query_interval = x;
/* x= MIN(1day, ttl/10, expire/10) */
x = 24 * 3600;
@ -1458,9 +1460,11 @@ set_tp_times(struct trust_anchor* tp, time_t rrsig_exp_interval,
if(rrsig_exp_interval/10 < x)
x = rrsig_exp_interval/10;
/* MAX(1hr, x) */
if(x < 3600)
tp->autr->retry_time = 3600;
else tp->autr->retry_time = x;
if(!autr_permit_small_holddown) {
if(x < 3600)
tp->autr->retry_time = 3600;
else tp->autr->retry_time = x;
} else tp->autr->retry_time = x;
if(qi != tp->autr->query_interval || rt != tp->autr->retry_time) {
*changed = 1;
@ -1959,8 +1963,12 @@ calc_next_probe(struct module_env* env, time_t wait)
{
/* make it random, 90-100% */
time_t rnd, rest;
if(wait < 3600)
wait = 3600;
if(!autr_permit_small_holddown) {
if(wait < 3600)
wait = 3600;
} else {
if(wait == 0) wait = 1;
}
rnd = wait/10;
rest = wait-rnd;
rnd = (time_t)ub_random_max(env->rnd, (long int)rnd);
@ -2349,6 +2357,8 @@ todo_probe(struct module_env* env, time_t* next)
if( (el=rbtree_first(&env->anchors->autr->probe)) == RBTREE_NULL) {
/* in case of revoked anchors */
lock_basic_unlock(&env->anchors->lock);
/* signal that there are no anchors to probe */
*next = 0;
return NULL;
}
tp = (struct trust_anchor*)el->key;
@ -2378,6 +2388,7 @@ autr_probe_timer(struct module_env* env)
struct trust_anchor* tp;
time_t next_probe = 3600;
int num = 0;
if(autr_permit_small_holddown) next_probe = 1;
verbose(VERB_ALGO, "autotrust probe timer callback");
/* while there are still anchors to probe */
while( (tp = todo_probe(env, &next_probe)) ) {
@ -2386,7 +2397,7 @@ autr_probe_timer(struct module_env* env)
num++;
}
regional_free_all(env->scratch);
if(num == 0)
if(next_probe == 0)
return 0; /* no trust points to probe */
verbose(VERB_ALGO, "autotrust probe timer %d callbacks done", num);
return next_probe;

2
external/unbound/validator/validator.c vendored
View File

@ -2769,7 +2769,7 @@ process_dnskey_response(struct module_qstate* qstate, struct val_qstate* vq,
vq->state = VAL_VALIDATE_STATE;
return;
}
downprot = 1;
downprot = qstate->env->cfg->harden_algo_downgrade;
vq->key_entry = val_verify_new_DNSKEYs(qstate->region, qstate->env,
ve, dnskey, vq->ds_rrset, downprot, &reason);

4
src/CMakeLists.txt
View File

@ -106,3 +106,7 @@ add_subdirectory(daemonizer)
add_subdirectory(daemon)
add_subdirectory(blockchain_utilities)
if(PER_BLOCK_CHECKPOINT)
add_subdirectory(blocks)
endif()

4
src/blockchain_db/CMakeLists.txt
View File

@ -31,7 +31,7 @@ set(blockchain_db_sources
lmdb/db_lmdb.cpp
)
if (NOT STATIC)
if (BERKELEY_DB)
set(blockchain_db_sources
${blockchain_db_sources}
berkeleydb/db_bdb.cpp
@ -46,7 +46,7 @@ set(blockchain_db_private_headers
lmdb/db_lmdb.h
)
if (NOT STATIC)
if (BERKELEY_DB)
set(blockchain_db_private_headers
${blockchain_db_private_headers}
berkeleydb/db_bdb.h

2805
src/blockchain_db/berkeleydb/db_bdb.cpp
View File

File diff suppressed because it is too large Load Diff

173
src/blockchain_db/berkeleydb/db_bdb.h
View File

@ -30,6 +30,11 @@
#include "blockchain_db/blockchain_db.h"
#include "cryptonote_protocol/blobdatatype.h" // for type blobdata
#include <unordered_map>
// ND: Enables multi-threaded bulk reads for when getting indices.
// TODO: Disabled for now, as it doesn't seem to provide noticeable improvements (??. Reason: TBD.
// #define BDB_BULK_CAN_THREAD
namespace cryptonote
{
@ -83,10 +88,145 @@ struct bdb_txn_safe
{
return &m_txn;
}
private:
DbTxn* m_txn;
};
// ND: Class to handle buffer management when doing bulk queries
// (DB_MULTIPLE). Allocates buffers then handles thread queuing
// so a fixed set of buffers can be used (instead of allocating
// every time a bulk query is needed).
template <typename T>
class bdb_safe_buffer
{
// limit the number of buffers to 8
const size_t MaxAllowedBuffers = 8;
public:
bdb_safe_buffer(size_t num_buffers, size_t count)
{
if(num_buffers > MaxAllowedBuffers)
num_buffers = MaxAllowedBuffers;
set_count(num_buffers);
for (size_t i = 0; i < num_buffers; i++)
m_buffers.push_back((T) malloc(sizeof(T) * count));
m_buffer_count = count;
}
~bdb_safe_buffer()
{
for (size_t i = 0; i < m_buffers.size(); i++)
{
if (m_buffers[i])
{
free(m_buffers[i]);
m_buffers[i] = nullptr;
}
}
m_buffers.resize(0);
}
T acquire_buffer()
{
std::unique_lock<std::mutex> lock(m_lock);
m_cv.wait(lock, [&]{ return m_count > 0; });
--m_count;
size_t index = -1;
for (size_t i = 0; i < m_open_slot.size(); i++)
{
if (m_open_slot[i])
{
m_open_slot[i] = false;
index = i;
break;
}
}
assert(index >= 0);
T buffer = m_buffers[index];
m_buffer_map.emplace(buffer, index);
return buffer;
}
void release_buffer(T buffer)
{
std::unique_lock<std::mutex> lock(m_lock);
assert(buffer != nullptr);
auto it = m_buffer_map.find(buffer);
if (it != m_buffer_map.end())
{
auto index = it->second;
assert(index < m_open_slot.size());
assert(m_open_slot[index] == false);
assert(m_count < m_open_slot.size());
++m_count;
m_open_slot[index] = true;
m_buffer_map.erase(it);
m_cv.notify_one();
}
}
size_t get_buffer_size() const
{
return m_buffer_count * sizeof(T);
}
size_t get_buffer_count() const
{
return m_buffer_count;
}
typedef T type;
private:
void set_count(size_t count)
{
assert(count > 0);
m_open_slot.resize(count, true);
m_count = count;
}
std::vector<T> m_buffers;
std::unordered_map<T, size_t> m_buffer_map;
std::condition_variable m_cv;
std::vector<bool> m_open_slot;
size_t m_count;
std::mutex m_lock;
size_t m_buffer_count;
};
template <typename T>
class bdb_safe_buffer_autolock
{
public:
bdb_safe_buffer_autolock(T &safe_buffer, typename T::type &buffer) :
m_safe_buffer(safe_buffer), m_buffer(nullptr)
{
m_buffer = m_safe_buffer.acquire_buffer();
buffer = m_buffer;
}
~bdb_safe_buffer_autolock()
{
if (m_buffer != nullptr)
{
m_safe_buffer.release_buffer(m_buffer);
m_buffer = nullptr;
}
}
private:
T &m_safe_buffer;
typename T::type m_buffer;
};
class BlockchainBDB : public BlockchainDB
{
public:
@ -155,12 +295,11 @@ public:
virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
virtual uint64_t get_random_output(const uint64_t& amount) const;
virtual uint64_t get_num_outputs(const uint64_t& amount) const;
virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const;
virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
virtual output_data_t get_output_key(const uint64_t& global_index) const;
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs);
virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const;
/**
@ -175,8 +314,11 @@ public:
tx_out get_output(const uint64_t& index) const;
virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
std::vector<tx_out_index> &tx_out_indices) const;
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
@ -191,13 +333,19 @@ public:
);
virtual void set_batch_transactions(bool batch_transactions);
virtual void batch_start();
virtual void batch_start(uint64_t batch_num_blocks=0);
virtual void batch_commit();
virtual void batch_stop();
virtual void batch_abort();
virtual void pop_block(block& blk, std::vector<transaction>& txs);
#if defined(BDB_BULK_CAN_THREAD)
virtual bool can_thread_bulk_indices() const { return true; }
#else
virtual bool can_thread_bulk_indices() const { return false; }
#endif
private:
virtual void add_block( const block& blk
, const size_t& block_size
@ -212,7 +360,7 @@ private:
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
virtual void remove_output(const tx_out& tx_output);
@ -225,6 +373,7 @@ private:
virtual void remove_spent_key(const crypto::key_image& k_image);
void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &global_indices);
/**
* @brief convert a tx output to a blob for storage
*
@ -251,9 +400,13 @@ private:
*
* @return the global index of the desired output
*/
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const;
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
void checkpoint_worker() const;
void check_open() const;
bool m_run_checkpoint;
std::unique_ptr<boost::thread> m_checkpoint_thread;
typedef bdb_safe_buffer<void *> bdb_safe_buffer_t;
bdb_safe_buffer_t m_buffer;
DbEnv* m_env;

39
src/blockchain_db/blockchain_db.cpp
View File

@ -56,24 +56,37 @@ void BlockchainDB::add_transaction(const crypto::hash& blk_hash, const transacti
tx_hash = *tx_hash_ptr;
}
for (const txin_v& tx_input : tx.vin)
{
if (tx_input.type() == typeid(txin_to_key))
{
add_spent_key(boost::get<txin_to_key>(tx_input).k_image);
}
else if (tx_input.type() == typeid(txin_gen))
{
/* nothing to do here */
}
else
{
LOG_PRINT_L1("Unsupported input type, removing key images and aborting transaction addition");
for (const txin_v& tx_input : tx.vin)
{
if (tx_input.type() == typeid(txin_to_key))
{
remove_spent_key(boost::get<txin_to_key>(tx_input).k_image);
}
}
return;
}
}
add_transaction_data(blk_hash, tx, tx_hash);
// iterate tx.vout using indices instead of C++11 foreach syntax because
// we need the index
if (tx.vout.size() != 0) // it may be technically possible for a tx to have no outputs
for (uint64_t i = 0; i < tx.vout.size(); ++i)
{
for (uint64_t i = 0; i < tx.vout.size(); ++i)
{
add_output(tx_hash, tx.vout[i], i);
}
for (const txin_v& tx_input : tx.vin)
{
if (tx_input.type() == typeid(txin_to_key))
{
add_spent_key(boost::get<txin_to_key>(tx_input).k_image);
}
}
add_output(tx_hash, tx.vout[i], i, tx.unlock_time);
}
}

31
src/blockchain_db/blockchain_db.h
View File

@ -104,7 +104,6 @@
* height get_tx_block_height(hash)
*
* Outputs:
* index get_random_output(amount)
* uint64_t get_num_outputs(amount)
* pub_key get_output_key(amount, index)
* tx_out get_output(tx_hash, index)
@ -138,6 +137,15 @@ namespace cryptonote
// typedef for convenience
typedef std::pair<crypto::hash, uint64_t> tx_out_index;
#pragma pack(push, 1)
struct output_data_t
{
crypto::public_key pubkey;
uint64_t unlock_time;
uint64_t height;
};
#pragma pack(pop)
/***********************************
* Exception Definitions
***********************************/
@ -279,7 +287,7 @@ private:
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx) = 0;
// tells the subclass to store an output
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index) = 0;
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time) = 0;
// tells the subclass to remove an output
virtual void remove_output(const tx_out& tx_output) = 0;
@ -313,7 +321,7 @@ protected:
mutable uint64_t time_tx_exists = 0;
uint64_t time_commit1 = 0;
bool m_auto_remove_logs = true;
public:
@ -357,7 +365,7 @@ public:
// release db lock
virtual void unlock() = 0;
virtual void batch_start() = 0;
virtual void batch_start(uint64_t batch_num_blocks=0) = 0;
virtual void batch_stop() = 0;
virtual void set_batch_transactions(bool) = 0;
@ -454,14 +462,12 @@ public:
// returns height of block that contains transaction with hash <h>
virtual uint64_t get_tx_block_height(const crypto::hash& h) const = 0;
// return global output index of a random output of amount <amount>
virtual uint64_t get_random_output(const uint64_t& amount) const = 0;
// returns the total number of outputs of amount <amount>
virtual uint64_t get_num_outputs(const uint64_t& amount) const = 0;
// return public key for output with global output amount <amount> and index <index>
virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const = 0;
virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index) = 0;
virtual output_data_t get_output_key(const uint64_t& global_index) const = 0;
// returns the output indexed by <index> in the transaction with hash <h>
virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const = 0;
@ -471,7 +477,11 @@ public:
// returns the transaction-local reference for the output with <amount> at <index>
// return type is pair of tx hash and index
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const = 0;
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) = 0;
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices) = 0;
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs) = 0;
virtual bool can_thread_bulk_indices() const = 0;
// return a vector of indices corresponding to the global output index for
// each output in the transaction with hash <h>
@ -483,7 +493,10 @@ public:
// returns true if key image <img> is present in spent key images storage
virtual bool has_key_image(const crypto::key_image& img) const = 0;
void set_auto_remove_logs(bool auto_remove) { m_auto_remove_logs = auto_remove; }
bool m_open;
mutable epee::critical_section m_synchronization_lock;
}; // class BlockchainDB

612
src/blockchain_db/lmdb/db_lmdb.cpp
View File

@ -28,8 +28,10 @@
#include "db_lmdb.h"
#include <boost/filesystem.hpp>
#include <boost/format.hpp>
#include <memory> // std::unique_ptr
#include <cstring> // memcpy
#include <random>
#include "cryptonote_core/cryptonote_format_utils.h"
#include "crypto/crypto.h"
@ -65,10 +67,19 @@ struct lmdb_cur
done = false;
}
~lmdb_cur() { close(); }
~lmdb_cur()
{
close();
}
operator MDB_cursor*() { return m_cur; }
operator MDB_cursor**() { return &m_cur; }
operator MDB_cursor*()
{
return m_cur;
}
operator MDB_cursor**()
{
return &m_cur;
}
void close()
{
@ -87,7 +98,8 @@ private:
template<typename T>
struct MDB_val_copy: public MDB_val
{
MDB_val_copy(const T &t): t_copy(t)
MDB_val_copy(const T &t) :
t_copy(t)
{
mv_size = sizeof (T);
mv_data = &t_copy;
@ -99,7 +111,8 @@ private:
template<>
struct MDB_val_copy<cryptonote::blobdata>: public MDB_val
{
MDB_val_copy(const cryptonote::blobdata &bd): data(new char[bd.size()])
MDB_val_copy(const cryptonote::blobdata &bd) :
data(new char[bd.size()])
{
memcpy(data.get(), bd.data(), bd.size());
mv_size = bd.size();
@ -109,7 +122,8 @@ private:
std::unique_ptr<char[]> data;
};
auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) {
auto compare_uint64 = [](const MDB_val *a, const MDB_val *b)
{
const uint64_t va = *(const uint64_t*)a->mv_data;
const uint64_t vb = *(const uint64_t*)b->mv_data;
if (va < vb) return -1;
@ -117,6 +131,20 @@ auto compare_uint64 = [](const MDB_val *a, const MDB_val *b) {
else return 1;
};
int compare_hash32(const MDB_val *a, const MDB_val *b)
{
uint32_t *va = (uint32_t*) a->mv_data;
uint32_t *vb = (uint32_t*) b->mv_data;
for (int n = 7; n >= 0; n--)
{
if (va[n] == vb[n])
continue;
return va[n] < vb[n] ? -1 : 1;
}
return 0;
}
const char* const LMDB_BLOCKS = "blocks";
const char* const LMDB_BLOCK_TIMESTAMPS = "block_timestamps";
const char* const LMDB_BLOCK_HEIGHTS = "block_heights";
@ -233,8 +261,29 @@ void mdb_txn_safe::allow_new_txns()
void BlockchainLMDB::do_resize()
void BlockchainLMDB::do_resize(uint64_t increase_size)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
CRITICAL_REGION_LOCAL(m_synchronization_lock);
const uint64_t add_size = 1LL << 30;
// check disk capacity
try
{
boost::filesystem::path path(m_folder);
boost::filesystem::space_info si = boost::filesystem::space(path);
if(si.available < add_size)
{
LOG_PRINT_RED_L0("!! WARNING: Insufficient free space to extend database !!: " << si.available / 1LL << 20L);
return;
}
}
catch(...)
{
// print something but proceed.
LOG_PRINT_YELLOW("Unable to query free disk space.", LOG_LEVEL_0);
}
MDB_envinfo mei;
mdb_env_info(m_env, &mei);
@ -243,7 +292,14 @@ void BlockchainLMDB::do_resize()
mdb_env_stat(m_env, &mst);
uint64_t new_mapsize = (double)mei.me_mapsize * RESIZE_FACTOR;
// add 1Gb per resize, instead of doing a percentage increase
uint64_t new_mapsize = (double) mei.me_mapsize + add_size;
// If given, use increase_size intead of above way of resizing.
// This is currently used for increasing by an estimated size at start of new
// batch txn.
if (increase_size > 0)
new_mapsize = mei.me_mapsize + increase_size;
new_mapsize += (new_mapsize % mst.ms_psize);
@ -265,15 +321,16 @@ void BlockchainLMDB::do_resize()
mdb_env_set_mapsize(m_env, new_mapsize);
LOG_PRINT_L0("LMDB Mapsize increased."
<< " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB"
<< ", New: " << new_mapsize / (1024 * 1024) << "MiB");
LOG_PRINT_GREEN("LMDB Mapsize increased." << " Old: " << mei.me_mapsize / (1024 * 1024) << "MiB" << ", New: " << new_mapsize / (1024 * 1024) << "MiB", LOG_LEVEL_0);
mdb_txn_safe::allow_new_txns();
}
bool BlockchainLMDB::need_resize() const
// threshold_size is used for batch transactions
bool BlockchainLMDB::need_resize(uint64_t threshold_size) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
#if defined(ENABLE_AUTO_RESIZE)
MDB_envinfo mei;
mdb_env_info(m_env, &mei);
@ -282,21 +339,130 @@ bool BlockchainLMDB::need_resize() const
mdb_env_stat(m_env, &mst);
// size_used doesn't include data yet to be committed, which can be
// significant size during batch transactions. For that, we estimate the size
// needed at the beginning of the batch transaction and pass in the
// additional size needed.
uint64_t size_used = mst.ms_psize * mei.me_last_pgno;
if ((double)size_used / mei.me_mapsize > RESIZE_PERCENT)
LOG_PRINT_L1("DB map size: " << mei.me_mapsize);
LOG_PRINT_L1("Space used: " << size_used);
LOG_PRINT_L1("Space remaining: " << mei.me_mapsize - size_used);
LOG_PRINT_L1("Size threshold: " << threshold_size);
float resize_percent_old = RESIZE_PERCENT;
LOG_PRINT_L1(boost::format("Percent used: %.04f Percent threshold: %.04f") % ((double)size_used/mei.me_mapsize) % resize_percent_old);
if (threshold_size > 0)
{
if (mei.me_mapsize - size_used < threshold_size)
{
LOG_PRINT_L1("Threshold met (size-based)");
return true;
}
else
return false;
}
std::mt19937 engine(std::random_device{}());
std::uniform_real_distribution<double> fdis(0.6, 0.9);
double resize_percent = fdis(engine);
if ((double)size_used / mei.me_mapsize > resize_percent)
{
LOG_PRINT_L1("Threshold met (percent-based)");
return true;
}
return false;
#else
return false;
#endif
}
void BlockchainLMDB::add_block( const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const crypto::hash& blk_hash
)
void BlockchainLMDB::check_and_resize_for_batch(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
LOG_PRINT_L1("[" << __func__ << "] " << "checking DB size");
const uint64_t min_increase_size = 128 * (1 << 20);
uint64_t threshold_size = 0;
uint64_t increase_size = 0;
if (batch_num_blocks > 0)
{
threshold_size = get_estimated_batch_size(batch_num_blocks);
LOG_PRINT_L1("calculated batch size: " << threshold_size);
// The increased DB size could be a multiple of threshold_size, a fixed
// size increase (> threshold_size), or other variations.
//
// Currently we use the greater of threshold size and a minimum size. The
// minimum size increase is used to avoid frequent resizes when the batch
// size is set to a very small numbers of blocks.
increase_size = (threshold_size > min_increase_size) ? threshold_size : min_increase_size;
LOG_PRINT_L1("increase size: " << increase_size);
}
// if threshold_size is 0 (i.e. number of blocks for batch not passed in), it
// will fall back to the percent-based threshold check instead of the
// size-based check
if (need_resize(threshold_size))
{
LOG_PRINT_L0("[batch] DB resize needed");
do_resize(increase_size);
}
}
uint64_t BlockchainLMDB::get_estimated_batch_size(uint64_t batch_num_blocks) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
uint64_t threshold_size = 0;
// batch size estimate * batch safety factor = final size estimate
// Takes into account "reasonable" block size increases in batch.
float batch_safety_factor = 1.7f;
// estimate of stored block expanded from raw block, including denormalization and db overhead.
// Note that this probably doesn't grow linearly with block size.
float db_expand_factor = 4.5f;
uint64_t num_prev_blocks = 500;
// For resizing purposes, allow for at least 4k average block size.
uint64_t min_block_size = 4 * 1024;
uint64_t block_stop = 0;
if (m_height > 1)
block_stop = m_height - 1;
uint64_t block_start = 0;
if (block_stop >= num_prev_blocks)
block_start = block_stop - num_prev_blocks + 1;
uint32_t num_blocks_used = 0;
uint64_t total_block_size = 0;
LOG_PRINT_L1("[" << __func__ << "] " << "m_height: " << m_height << " block_start: " << block_start << " block_stop: " << block_stop);
size_t avg_block_size = 0;
if (m_height == 0)
{
LOG_PRINT_L1("No existing blocks to check for average block size");
}
else
{
for (uint64_t block_num = block_start; block_num <= block_stop; ++block_num)
{
uint32_t block_size = get_block_size(block_num);
total_block_size += block_size;
// Track number of blocks being totalled here instead of assuming, in case
// some blocks were to be skipped for being outliers.
++num_blocks_used;
}
avg_block_size = total_block_size / num_blocks_used;
LOG_PRINT_L1("average block size across recent " << num_blocks_used << " blocks: " << avg_block_size);
}
if (avg_block_size < min_block_size)
avg_block_size = min_block_size;
LOG_PRINT_L1("estimated average block size for batch: " << avg_block_size);
threshold_size = avg_block_size * db_expand_factor * batch_num_blocks;
threshold_size = threshold_size * batch_safety_factor;
return threshold_size;
}
void BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
const crypto::hash& blk_hash)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@ -447,7 +613,7 @@ void BlockchainLMDB::remove_transaction_data(const crypto::hash& tx_hash, const
}
void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index)
void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
@ -476,10 +642,15 @@ void BlockchainLMDB::add_output(const crypto::hash& tx_hash, const tx_out& tx_ou
if (tx_output.target.type() == typeid(txout_to_key))
{
MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key);
result = mdb_put(*m_write_txn, m_output_keys, &k, &val_pubkey, 0);
if (result)
throw0(DB_ERROR(std::string("Failed to add output pubkey to db transaction: ").append(mdb_strerror(result)).c_str()));
output_data_t od;
od.pubkey = boost::get < txout_to_key > (tx_output.target).key;
od.unlock_time = unlock_time;
od.height = m_height;
MDB_val_copy<output_data_t> data(od);
//MDB_val_copy<crypto::public_key> val_pubkey(boost::get<txout_to_key>(tx_output.target).key);
if (mdb_put(*m_write_txn, m_output_keys, &k, &data, 0))
throw0(DB_ERROR("Failed to add output pubkey to db transaction"));
}
@ -710,47 +881,17 @@ tx_out BlockchainLMDB::output_from_blob(const blobdata& blob) const
return o;
}
uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index) const
uint64_t BlockchainLMDB::get_output_global_index(const uint64_t& amount, const uint64_t& index)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
lmdb_cur cur(txn, m_output_amounts);
MDB_val_copy<uint64_t> k(amount);
MDB_val v;
auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
std::vector <uint64_t> offsets;
std::vector <uint64_t> global_indices;
offsets.push_back(index);
get_output_global_indices(amount, offsets, global_indices);
if (!global_indices.size())
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
if (num_elems <= index)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
for (uint64_t i = 0; i < index; ++i)
{
mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
}
mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
uint64_t glob_index = *(const uint64_t*)v.mv_data;
cur.close();
txn.commit();
return glob_index;
return global_indices[0];
}
void BlockchainLMDB::check_open() const
@ -805,8 +946,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
// check for existing LMDB files in base directory
boost::filesystem::path old_files = direc.parent_path();
if (boost::filesystem::exists(old_files / "data.mdb") ||
boost::filesystem::exists(old_files / "lock.mdb"))
if (boost::filesystem::exists(old_files / "data.mdb") || boost::filesystem::exists(old_files / "lock.mdb"))
{
LOG_PRINT_L0("Found existing LMDB files in " << old_files.string());
LOG_PRINT_L0("Move data.mdb and/or lock.mdb to " << filename << ", or delete them, and then restart");
@ -872,7 +1012,7 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
lmdb_db_open(txn, LMDB_OUTPUT_TXS, MDB_INTEGERKEY | MDB_CREATE, m_output_txs, "Failed to open db handle for m_output_txs");
lmdb_db_open(txn, LMDB_OUTPUT_INDICES, MDB_INTEGERKEY | MDB_CREATE, m_output_indices, "Failed to open db handle for m_output_indices");
lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
lmdb_db_open(txn, LMDB_OUTPUT_AMOUNTS, MDB_INTEGERKEY | MDB_DUPSORT | MDB_DUPFIXED | MDB_CREATE, m_output_amounts, "Failed to open db handle for m_output_amounts");
lmdb_db_open(txn, LMDB_OUTPUT_KEYS, MDB_INTEGERKEY | MDB_CREATE, m_output_keys, "Failed to open db handle for m_output_keys");
/*************** not used, but kept for posterity
@ -884,6 +1024,11 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
mdb_set_dupsort(txn, m_output_amounts, compare_uint64);
mdb_set_dupsort(txn, m_tx_outputs, compare_uint64);
mdb_set_compare(txn, m_spent_keys, compare_hash32);
mdb_set_compare(txn, m_block_heights, compare_hash32);
mdb_set_compare(txn, m_txs, compare_hash32);
mdb_set_compare(txn, m_tx_unlocks, compare_hash32);
mdb_set_compare(txn, m_tx_heights, compare_hash32);
// get and keep current height
MDB_stat db_stats;
@ -897,6 +1042,34 @@ void BlockchainLMDB::open(const std::string& filename, const int mdb_flags)
throw0(DB_ERROR("Failed to query m_output_indices"));
m_num_outputs = db_stats.ms_entries;
// ND: This "new" version of the lmdb database is incompatible with
// the previous version. Ensure that the output_keys database is
// sizeof(output_data_t) in length. Otherwise, inform user and
// terminate.
if(m_height > 0)
{
MDB_val_copy<uint64_t> k(0);
MDB_val v;
auto get_result = mdb_get(txn, m_output_keys, &k, &v);
if(get_result != MDB_SUCCESS)
{
txn.abort();
m_open = false;
return;
}
// LOG_PRINT_L0("Output keys size: " << v.mv_size);
if(v.mv_size != sizeof(output_data_t))
{
txn.abort();
mdb_env_close(m_env);
m_open = false;
LOG_PRINT_RED_L0("Existing lmdb database is incompatible with this version.");
LOG_PRINT_RED_L0("Please delete the existing database and resync.");
return;
}
}
// commit the transaction
txn.commit();
@ -1464,19 +1637,6 @@ uint64_t BlockchainLMDB::get_tx_block_height(const crypto::hash& h) const
return *(const uint64_t*)result.mv_data;
}
//FIXME: make sure the random method used here is appropriate
uint64_t BlockchainLMDB::get_random_output(const uint64_t& amount) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
uint64_t num_outputs = get_num_outputs(amount);
if (num_outputs == 0)
throw1(OUTPUT_DNE("Attempting to get a random output for an amount, but none exist"));
return crypto::rand<uint64_t>() % num_outputs;
}
uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
@ -1506,26 +1666,33 @@ uint64_t BlockchainLMDB::get_num_outputs(const uint64_t& amount) const
return num_elems;
}
crypto::public_key BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index) const
output_data_t BlockchainLMDB::get_output_key(const uint64_t &global_index) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
uint64_t glob_index = get_output_global_index(amount, index);
mdb_txn_safe txn;
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
MDB_val_copy<uint64_t> k(glob_index);
MDB_val_copy<uint64_t> k(global_index);
MDB_val v;
auto get_result = mdb_get(txn, m_output_keys, &k, &v);
if (get_result == MDB_NOTFOUND)
throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist"));
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
txn.commit();
return *(output_data_t *) v.mv_data;
}
return *(crypto::public_key*)v.mv_data;
output_data_t BlockchainLMDB::get_output_key(const uint64_t& amount, const uint64_t& index)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
uint64_t glob_index = get_output_global_index(amount, index);
return get_output_key(glob_index);
}
tx_out BlockchainLMDB::get_output(const crypto::hash& h, const uint64_t& index) const
@ -1633,53 +1800,17 @@ tx_out_index BlockchainLMDB::get_output_tx_and_index_from_global(const uint64_t&
return tx_out_index(tx_hash, *(const uint64_t *)v.mv_data);
}
tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const
tx_out_index BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const uint64_t& index)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
lmdb_cur cur(*txn_ptr, m_output_amounts);
MDB_val_copy<uint64_t> k(amount);
MDB_val v;
auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
std::vector < uint64_t > offsets;
std::vector<tx_out_index> indices;
offsets.push_back(index);
get_output_tx_and_index(amount, offsets, indices);
if (!indices.size())
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
if (num_elems <= index)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
for (uint64_t i = 0; i < index; ++i)
{
mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
}
mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
uint64_t glob_index = *(const uint64_t*)v.mv_data;
cur.close();
if (! m_batch_active)
txn.commit();
return get_output_tx_and_index_from_global(glob_index);
return indices[0];
}
std::vector<uint64_t> BlockchainLMDB::get_tx_output_indices(const crypto::hash& h) const
@ -1820,7 +1951,8 @@ bool BlockchainLMDB::has_key_image(const crypto::key_image& img) const
return false;
}
void BlockchainLMDB::batch_start()
// batch_num_blocks: (optional) Used to check if resize needed before batch transaction starts.
void BlockchainLMDB::batch_start(uint64_t batch_num_blocks)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
if (! m_batch_transactions)
@ -1833,6 +1965,8 @@ void BlockchainLMDB::batch_start()
throw0(DB_ERROR("batch transaction attempted, but m_write_txn already in use"));
check_open();
check_and_resize_for_batch(batch_num_blocks);
m_write_batch_txn = new mdb_txn_safe();
// NOTE: need to make sure it's destroyed properly when done
@ -1915,19 +2049,16 @@ void BlockchainLMDB::set_batch_transactions(bool batch_transactions)
LOG_PRINT_L3("batch transactions " << (m_batch_transactions ? "enabled" : "disabled"));
}
uint64_t BlockchainLMDB::add_block( const block& blk
, const size_t& block_size
, const difficulty_type& cumulative_difficulty
, const uint64_t& coins_generated
, const std::vector<transaction>& txs
)
uint64_t BlockchainLMDB::add_block(const block& blk, const size_t& block_size, const difficulty_type& cumulative_difficulty, const uint64_t& coins_generated,
const std::vector<transaction>& txs)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
if (m_height % 1000 == 0)
{
if (need_resize())
// for batch mode, DB resize check is done at start of batch transaction
if (! m_batch_active && need_resize())
{
LOG_PRINT_L0("LMDB memory map needs resized, doing that now.");
do_resize();
@ -2001,4 +2132,219 @@ void BlockchainLMDB::pop_block(block& blk, std::vector<transaction>& txs)
--m_height;
}
void BlockchainLMDB::get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
std::vector<tx_out_index> &tx_out_indices) const
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
tx_out_indices.clear();
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
for (const uint64_t &index : global_indices)
{
MDB_val_copy<uint64_t> k(index);
MDB_val v;
auto get_result = mdb_get(*txn_ptr, m_output_txs, &k, &v);
if (get_result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx hash"));
crypto::hash tx_hash = *(crypto::hash*) v.mv_data;
get_result = mdb_get(*txn_ptr, m_output_indices, &k, &v);
if (get_result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("output with given index not in db"));
else if (get_result)
throw0(DB_ERROR("DB error attempting to fetch output tx index"));
auto result = tx_out_index(tx_hash, *(const uint64_t *) v.mv_data);
tx_out_indices.push_back(result);
}
if (!m_batch_active)
txn.commit();
}
void BlockchainLMDB::get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets,
std::vector<uint64_t> &global_indices)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
TIME_MEASURE_START(txx);
check_open();
global_indices.clear();
uint64_t max = 0;
for (const uint64_t &index : offsets)
{
if (index > max)
max = index;
}
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if(m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
lmdb_cur cur(*txn_ptr, m_output_amounts);
MDB_val_copy<uint64_t> k(amount);
MDB_val v;
auto result = mdb_cursor_get(cur, &k, &v, MDB_SET);
if (result == MDB_NOTFOUND)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but amount not found"));
else if (result)
throw0(DB_ERROR("DB error attempting to get an output"));
size_t num_elems = 0;
mdb_cursor_count(cur, &num_elems);
if (max <= 1 && num_elems <= max)
throw1(OUTPUT_DNE("Attempting to get an output index by amount and amount index, but output not found"));
uint64_t t_dbmul = 0;
uint64_t t_dbscan = 0;
if (max <= 1)
{
for (const uint64_t& index : offsets)
{
mdb_cursor_get(cur, &k, &v, MDB_FIRST_DUP);
for (uint64_t i = 0; i < index; ++i)
{
mdb_cursor_get(cur, &k, &v, MDB_NEXT_DUP);
}
mdb_cursor_get(cur, &k, &v, MDB_GET_CURRENT);
uint64_t glob_index = *(const uint64_t*) v.mv_data;
LOG_PRINT_L3("Amount: " << amount << " M0->v: " << glob_index);
global_indices.push_back(glob_index);
}
}
else
{
uint32_t curcount = 0;
uint32_t blockstart = 0;
for (const uint64_t& index : offsets)
{
if (index >= num_elems)
{
LOG_PRINT_L1("Index: " << index << " Elems: " << num_elems << " partial results found for get_output_tx_and_index");
break;
}
while (index >= curcount)
{
TIME_MEASURE_START(db1);
if (mdb_cursor_get(cur, &k, &v, curcount == 0 ? MDB_GET_MULTIPLE : MDB_NEXT_MULTIPLE) != 0)
{
// allow partial results
result = false;
break;
}
int count = v.mv_size / sizeof(uint64_t);
blockstart = curcount;
curcount += count;
TIME_MEASURE_FINISH(db1);
t_dbmul += db1;
}
LOG_PRINT_L3("Records returned: " << curcount << " Index: " << index);
TIME_MEASURE_START(db2);
uint64_t actual_index = index - blockstart;
uint64_t glob_index = ((const uint64_t*) v.mv_data)[actual_index];
LOG_PRINT_L3("Amount: " << amount << " M1->v: " << glob_index);
global_indices.push_back(glob_index);
TIME_MEASURE_FINISH(db2);
t_dbscan += db2;
}
}
cur.close();
if(!m_batch_active)
txn.commit();
TIME_MEASURE_FINISH(txx);
LOG_PRINT_L3("txx: " << txx << " db1: " << t_dbmul << " db2: " << t_dbscan);
}
void BlockchainLMDB::get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
TIME_MEASURE_START(db3);
check_open();
outputs.clear();
std::vector <uint64_t> global_indices;
get_output_global_indices(amount, offsets, global_indices);
if (global_indices.size() > 0)
{
mdb_txn_safe txn;
mdb_txn_safe* txn_ptr = &txn;
if (m_batch_active)
txn_ptr = m_write_txn;
else
{
if (mdb_txn_begin(m_env, NULL, MDB_RDONLY, txn))
throw0(DB_ERROR("Failed to create a transaction for the db"));
}
for (const uint64_t &index : global_indices)
{
MDB_val_copy<uint64_t> k(index);
MDB_val v;
auto get_result = mdb_get(*txn_ptr, m_output_keys, &k, &v);
if (get_result != 0)
throw0(DB_ERROR("Attempting to get output pubkey by global index, but key does not exist"));
else if (get_result)
throw0(DB_ERROR("Error attempting to retrieve an output pubkey from the db"));
output_data_t data = *(output_data_t *) v.mv_data;
outputs.push_back(data);
}
if (!m_batch_active)
txn.commit();
}
TIME_MEASURE_FINISH(db3);
LOG_PRINT_L3("db3: " << db3);
}
void BlockchainLMDB::get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices)
{
LOG_PRINT_L3("BlockchainLMDB::" << __func__);
check_open();
indices.clear();
std::vector <uint64_t> global_indices;
get_output_global_indices(amount, offsets, global_indices);
TIME_MEASURE_START(db3);
if(global_indices.size() > 0)
{
get_output_tx_and_index_from_global(global_indices, indices);
}
TIME_MEASURE_FINISH(db3);
LOG_PRINT_L3("db3: " << db3);
}
} // namespace cryptonote

40
src/blockchain_db/lmdb/db_lmdb.h
View File

@ -33,6 +33,8 @@
#include <lmdb.h>
#define ENABLE_AUTO_RESIZE
namespace cryptonote
{
@ -155,11 +157,11 @@ public:
virtual uint64_t get_tx_block_height(const crypto::hash& h) const;
virtual uint64_t get_random_output(const uint64_t& amount) const;
virtual uint64_t get_num_outputs(const uint64_t& amount) const;
virtual crypto::public_key get_output_key(const uint64_t& amount, const uint64_t& index) const;
virtual output_data_t get_output_key(const uint64_t& amount, const uint64_t& index);
virtual output_data_t get_output_key(const uint64_t& global_index) const;
virtual void get_output_key(const uint64_t &amount, const std::vector<uint64_t> &offsets, std::vector<output_data_t> &outputs);
virtual tx_out get_output(const crypto::hash& h, const uint64_t& index) const;
@ -175,8 +177,12 @@ public:
tx_out get_output(const uint64_t& index) const;
virtual tx_out_index get_output_tx_and_index_from_global(const uint64_t& index) const;
virtual void get_output_tx_and_index_from_global(const std::vector<uint64_t> &global_indices,
std::vector<tx_out_index> &tx_out_indices) const;
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index) const;
virtual tx_out_index get_output_tx_and_index(const uint64_t& amount, const uint64_t& index);
virtual void get_output_tx_and_index(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<tx_out_index> &indices);
virtual void get_output_global_indices(const uint64_t& amount, const std::vector<uint64_t> &offsets, std::vector<uint64_t> &indices);
virtual std::vector<uint64_t> get_tx_output_indices(const crypto::hash& h) const;
virtual std::vector<uint64_t> get_tx_amount_output_indices(const crypto::hash& h) const;
@ -191,17 +197,20 @@ public:
);
virtual void set_batch_transactions(bool batch_transactions);
virtual void batch_start();
virtual void batch_start(uint64_t batch_num_blocks=0);
virtual void batch_commit();
virtual void batch_stop();
virtual void batch_abort();
virtual void pop_block(block& blk, std::vector<transaction>& txs);
virtual bool can_thread_bulk_indices() const { return true; }
private:
void do_resize();
void do_resize(uint64_t size_increase=0);
bool need_resize() const;
bool need_resize(uint64_t threshold_size=0) const;
void check_and_resize_for_batch(uint64_t batch_num_blocks);
uint64_t get_estimated_batch_size(uint64_t batch_num_blocks) const;
virtual void add_block( const block& blk
, const size_t& block_size
@ -216,7 +225,7 @@ private:
virtual void remove_transaction_data(const crypto::hash& tx_hash, const transaction& tx);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index);
virtual void add_output(const crypto::hash& tx_hash, const tx_out& tx_output, const uint64_t& local_index, const uint64_t unlock_time);
virtual void remove_output(const tx_out& tx_output);
@ -255,7 +264,7 @@ private:
*
* @return the global index of the desired output
*/
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index) const;
uint64_t get_output_global_index(const uint64_t& amount, const uint64_t& index);
void check_open() const;
@ -292,9 +301,18 @@ private:
bool m_batch_transactions; // support for batch transactions
bool m_batch_active; // whether batch transaction is in progress
constexpr static uint64_t DEFAULT_MAPSIZE = 1 << 30;
#if defined(__arm__)
// force a value so it can compile with 32-bit ARM
constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 31;
#else
#if defined(ENABLE_AUTO_RESIZE)
constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 30;
#else
constexpr static uint64_t DEFAULT_MAPSIZE = 1LL << 33;
#endif
#endif
constexpr static float RESIZE_PERCENT = 0.8f;
constexpr static float RESIZE_FACTOR = 1.5f;
};
} // namespace cryptonote

14
src/blockchain_utilities/README.md
View File

@ -4,13 +4,19 @@ Copyright (c) 2014-2015, The Monero Project
## Introduction
The blockchain utilities allow one to convert an old style blockchain.bin file
to a new style database. There are two ways to upgrade an old style blockchain:
The recommended way is to run a `blockchain_export`, then `blockchain_import`.
The other way is to run `blockchain_converter`. In both cases, you will be left
with a new style blockchain.
For importing into the LMDB database, compile with `DATABASE=lmdb`
e.g.
`DATABASE=lmdb make release`
This is also the default compile setting on the blockchain branch.
This is also the default compile setting on the master branch.
By default, the exporter will use the original in-memory database (blockchain.bin) as its source.
This default is to make migrating to an LMDB database easy, without having to recompile anything.
@ -30,7 +36,8 @@ This loads the existing blockchain, for whichever database type it was compiled
`$ blockchain_import`
This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` into the current database.
This imports blocks from `$MONERO_DATA_DIR/export/blockchain.raw` (exported using the `blockchain_export` tool as described above)
into the current database.
Defaults: `--batch on`, `--batch size 20000`, `--verify on`
@ -38,6 +45,9 @@ Batch size refers to number of blocks and can be adjusted for performance based
Verification should only be turned off if importing from a trusted blockchain.
If you encounter an error like "resizing not supported in batch mode", you can just re-run
the `blockchain_import` command again, and it will restart from where it left off.
```bash
## use default settings to import blockchain.raw into database
$ blockchain_import

4
src/blockchain_utilities/blockchain_converter.cpp
View File

@ -236,7 +236,7 @@ int main(int argc, char* argv[])
}
if (opt_batch)
blockchain->batch_start();
blockchain->batch_start(db_batch_size);
uint64_t i = 0;
for (i = start_block; i < end_block + 1; ++i)
{
@ -277,7 +277,7 @@ int main(int argc, char* argv[])
std::cout << "\r \r";
std::cout << "[- batch commit at height " << i + 1 << " -]" << ENDL;
blockchain->batch_stop();
blockchain->batch_start();
blockchain->batch_start(db_batch_size);
std::cout << ENDL;
blockchain->show_stats();
}

15
src/blockchain_utilities/blockchain_export.cpp
View File

@ -39,13 +39,14 @@ int main(int argc, char* argv[])
{
uint32_t log_level = 0;
uint64_t block_stop = 0;
std::string import_filename = BLOCKCHAIN_RAW;
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
boost::filesystem::path output_file_path;
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<std::string> arg_output_file = {"output-file", "Specify output file", "", true};
const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<bool> arg_testnet_on = {
@ -57,6 +58,7 @@ int main(int argc, char* argv[])
command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_output_file);
command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_block_stop);
@ -97,9 +99,12 @@ int main(int argc, char* argv[])
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
m_config_folder = command_line::get_arg(vm, data_dir_arg);
boost::filesystem::path output_dir {m_config_folder};
output_dir /= "export";
LOG_PRINT_L0("Export directory: " << output_dir.string());
if (command_line::has_arg(vm, arg_output_file))
output_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_output_file));
else
output_file_path = boost::filesystem::path(m_config_folder) / "export" / BLOCKCHAIN_RAW;
LOG_PRINT_L0("Export output file: " << output_file_path.string());
// If we wanted to use the memory pool, we would set up a fake_core.
@ -151,7 +156,7 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("Exporting blockchain raw data...");
BootstrapFile bootstrap;
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_dir, block_stop);
r = bootstrap.store_blockchain_raw(core_storage, NULL, output_file_path, block_stop);
CHECK_AND_ASSERT_MES(r, false, "Failed to export blockchain raw data");
LOG_PRINT_L0("Blockchain raw data exported OK");
}

41
src/blockchain_utilities/blockchain_import.cpp
View File

@ -173,7 +173,7 @@ int pop_blocks(FakeCore& simple_core, int num_blocks)
}
template <typename FakeCore>
int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint64_t block_stop=0)
int import_from_file(FakeCore& simple_core, const std::string& import_file_path, uint64_t block_stop=0)
{
#if !defined(BLOCKCHAIN_DB)
static_assert(std::is_same<fake_core_memory, FakeCore>::value || std::is_same<fake_core_lmdb, FakeCore>::value,
@ -188,11 +188,11 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
simple_core.m_storage.get_db().reset_stats();
}
#endif
boost::filesystem::path raw_file_path(import_file_path);
boost::filesystem::path fs_import_file_path(import_file_path);
boost::system::error_code ec;
if (!boost::filesystem::exists(raw_file_path, ec))
if (!boost::filesystem::exists(fs_import_file_path, ec))
{
LOG_PRINT_L0("bootstrap file not found: " << raw_file_path);
LOG_PRINT_L0("bootstrap file not found: " << fs_import_file_path);
return false;
}
@ -254,7 +254,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
}
if (use_batch)
simple_core.batch_start();
simple_core.batch_start(db_batch_size);
LOG_PRINT_L0("Reading blockchain from bootstrap file...");
std::cout << ENDL;
@ -482,7 +482,7 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
// zero-based height
std::cout << ENDL << "[- batch commit at height " << h-1 << " -]" << ENDL;
simple_core.batch_stop();
simple_core.batch_start();
simple_core.batch_start(db_batch_size);
std::cout << ENDL;
#if !defined(BLOCKCHAIN_DB) || (BLOCKCHAIN_DB == DB_LMDB)
simple_core.m_storage.get_db().show_stats();
@ -526,7 +526,6 @@ int import_from_file(FakeCore& simple_core, std::string& import_file_path, uint6
int main(int argc, char* argv[])
{
std::string import_filename = BLOCKCHAIN_RAW;
#if defined(BLOCKCHAIN_DB) && (BLOCKCHAIN_DB == DB_MEMORY)
std::string default_db_engine = "memory";
#else
@ -536,14 +535,16 @@ int main(int argc, char* argv[])
uint32_t log_level = LOG_LEVEL_0;
uint64_t num_blocks = 0;
uint64_t block_stop = 0;
std::string dirname;
std::string m_config_folder;
std::string db_arg_str;
boost::filesystem::path default_data_path {tools::get_default_data_dir()};
boost::filesystem::path default_testnet_data_path {default_data_path / "testnet"};
std::string import_file_path;
po::options_description desc_cmd_only("Command line options");
po::options_description desc_cmd_sett("Command line options and settings options");
const command_line::arg_descriptor<std::string> arg_input_file = {"input-file", "Specify input file", "", true};
const command_line::arg_descriptor<uint32_t> arg_log_level = {"log-level", "", log_level};
const command_line::arg_descriptor<uint64_t> arg_block_stop = {"block-stop", "Stop at block number", block_stop};
const command_line::arg_descriptor<uint64_t> arg_batch_size = {"batch-size", "", db_batch_size};
@ -571,6 +572,7 @@ int main(int argc, char* argv[])
command_line::add_arg(desc_cmd_sett, command_line::arg_data_dir, default_data_path.string());
command_line::add_arg(desc_cmd_sett, command_line::arg_testnet_data_dir, default_testnet_data_path.string());
command_line::add_arg(desc_cmd_sett, arg_input_file);
command_line::add_arg(desc_cmd_sett, arg_testnet_on);
command_line::add_arg(desc_cmd_sett, arg_log_level);
command_line::add_arg(desc_cmd_sett, arg_database);
@ -643,7 +645,7 @@ int main(int argc, char* argv[])
opt_testnet = command_line::get_arg(vm, arg_testnet_on);
auto data_dir_arg = opt_testnet ? command_line::arg_testnet_data_dir : command_line::arg_data_dir;
dirname = command_line::get_arg(vm, data_dir_arg);
m_config_folder = command_line::get_arg(vm, data_dir_arg);
db_arg_str = command_line::get_arg(vm, arg_database);
log_space::get_set_log_detalisation_level(true, log_level);
@ -651,10 +653,15 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("Starting...");
LOG_PRINT_L0("Setting log level = " << log_level);
boost::filesystem::path file_path {dirname};
std::string import_file_path;
boost::filesystem::path fs_import_file_path;
if (command_line::has_arg(vm, arg_input_file))
fs_import_file_path = boost::filesystem::path(command_line::get_arg(vm, arg_input_file));
else
fs_import_file_path = boost::filesystem::path(m_config_folder) / "export" / BLOCKCHAIN_RAW;
import_file_path = fs_import_file_path.string();
import_file_path = (file_path / "export" / import_filename).string();
if (command_line::has_arg(vm, arg_count_blocks))
{
BootstrapFile bootstrap;
@ -694,7 +701,7 @@ int main(int argc, char* argv[])
LOG_PRINT_L0("testnet: " << std::boolalpha << opt_testnet << std::noboolalpha);
LOG_PRINT_L0("bootstrap file path: " << import_file_path);
LOG_PRINT_L0("database path: " << file_path.string());
LOG_PRINT_L0("database path: " << m_config_folder);
try
{
@ -710,12 +717,12 @@ int main(int argc, char* argv[])
#if !defined(BLOCKCHAIN_DB)
if (db_engine == "lmdb")
{
fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags);
fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
import_from_file(simple_core, import_file_path, block_stop);
}
else if (db_engine == "memory")
{
fake_core_memory simple_core(dirname, opt_testnet);
fake_core_memory simple_core(m_config_folder, opt_testnet);
import_from_file(simple_core, import_file_path, block_stop);
}
else
@ -732,9 +739,9 @@ int main(int argc, char* argv[])
return 1;
}
#if BLOCKCHAIN_DB == DB_LMDB
fake_core_lmdb simple_core(dirname, opt_testnet, opt_batch, mdb_flags);
fake_core_lmdb simple_core(m_config_folder, opt_testnet, opt_batch, mdb_flags);
#else
fake_core_memory simple_core(dirname, opt_testnet);
fake_core_memory simple_core(m_config_folder, opt_testnet);
#endif
if (! vm["pop-blocks"].defaulted())

18
src/blockchain_utilities/bootstrap_file.cpp
View File

@ -50,8 +50,9 @@ namespace
bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path)
bool BootstrapFile::open_writer(const boost::filesystem::path& file_path)
{
const boost::filesystem::path dir_path = file_path.parent_path();
if (boost::filesystem::exists(dir_path))
{
if (!boost::filesystem::is_directory(dir_path))
@ -69,7 +70,6 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path)
}
}
std::string file_path = (dir_path / BLOCKCHAIN_RAW).string();
m_raw_data_file = new std::ofstream();
bool do_initialize_file = false;
@ -83,15 +83,15 @@ bool BootstrapFile::open_writer(const boost::filesystem::path& dir_path)
}
else
{
num_blocks = count_blocks(file_path);
num_blocks = count_blocks(file_path.string());
LOG_PRINT_L0("appending to existing file with height: " << num_blocks-1 << " total blocks: " << num_blocks);
}
m_height = num_blocks;
if (do_initialize_file)
m_raw_data_file->open(file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc);
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::trunc);
else
m_raw_data_file->open(file_path, std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
m_raw_data_file->open(file_path.string(), std::ios_base::binary | std::ios_base::out | std::ios::app | std::ios::ate);
if (m_raw_data_file->fail())
return false;
@ -286,9 +286,9 @@ bool BootstrapFile::close()
#if SOURCE_DB == DB_MEMORY
bool BootstrapFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_stop)
bool BootstrapFile::store_blockchain_raw(blockchain_storage* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
#else
bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_dir, uint64_t requested_block_stop)
bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_memory_pool* _tx_pool, boost::filesystem::path& output_file, uint64_t requested_block_stop)
#endif
{
uint64_t num_blocks_written = 0;
@ -297,7 +297,7 @@ bool BootstrapFile::store_blockchain_raw(Blockchain* _blockchain_storage, tx_mem
m_tx_pool = _tx_pool;
uint64_t progress_interval = 100;
LOG_PRINT_L0("Storing blocks raw data...");
if (!BootstrapFile::open_writer(output_dir))
if (!BootstrapFile::open_writer(output_file))
{
LOG_PRINT_RED_L0("failed to open raw file for write");
return false;
@ -454,7 +454,7 @@ uint64_t BootstrapFile::count_blocks(const std::string& import_file_path)
str1.assign(buf1, sizeof(chunk_size));
if (! ::serialization::parse_binary(str1, chunk_size))
throw std::runtime_error("Error in deserialization of chunk_size");
LOG_PRINT_L1("chunk_size: " << chunk_size);
LOG_PRINT_L3("chunk_size: " << chunk_size);
if (chunk_size > BUFFER_SIZE)
{

Some files were not shown because too many files have changed in this diff Show More