diff -Nru mercurial-4.0.2~trusty1/contrib/all-revsets.txt mercurial-4.1~rc~trusty1/contrib/all-revsets.txt --- mercurial-4.0.2~trusty1/contrib/all-revsets.txt 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/all-revsets.txt 2017-01-18 16:54:51.000000000 +0000 @@ -8,7 +8,7 @@ # # Please update this file with any revsets you use for benchmarking a change so # that future contributors can easily find and retest it when doing further -# modification. Feel free to highlight interresting variants if needed. +# modification. Feel free to highlight interesting variants if needed. ## Revset from this section are all extracted from changelog when this file was diff -Nru mercurial-4.0.2~trusty1/contrib/asv.conf.json mercurial-4.1~rc~trusty1/contrib/asv.conf.json --- mercurial-4.0.2~trusty1/contrib/asv.conf.json 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/asv.conf.json 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,13 @@ +{ + "version": 1, + "project": "mercurial", + "project_url": "https://mercurial-scm.org/", + "repo": "..", + "branches": ["default", "stable"], + "environment_type": "virtualenv", + "show_commit_url": "https://www.mercurial-scm.org/repo/hg/rev/", + "benchmark_dir": "benchmarks", + "env_dir": "../.asv/env", + "results_dir": "../.asv/results", + "html_dir": "../.asv/html" +} diff -Nru mercurial-4.0.2~trusty1/contrib/benchmarks/__init__.py mercurial-4.1~rc~trusty1/contrib/benchmarks/__init__.py --- mercurial-4.0.2~trusty1/contrib/benchmarks/__init__.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/benchmarks/__init__.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,113 @@ +# __init__.py - asv benchmark suite +# +# Copyright 2016 Logilab SA +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +# "historical portability" policy of contrib/benchmarks: +# +# We have to make this code work correctly with current mercurial stable branch +# and if possible with reasonable cost with early Mercurial versions. + +'''ASV (https://asv.readthedocs.io) benchmark suite + +Benchmark are parameterized against reference repositories found in the +directory pointed by the REPOS_DIR environment variable. + +Invocation example: + + $ export REPOS_DIR=~/hgperf/repos + # run suite on given revision + $ asv --config contrib/asv.conf.json run REV + # run suite on new changesets found in stable and default branch + $ asv --config contrib/asv.conf.json run NEW + # display a comparative result table of benchmark results between two given + # revisions + $ asv --config contrib/asv.conf.json compare REV1 REV2 + # compute regression detection and generate ASV static website + $ asv --config contrib/asv.conf.json publish + # serve the static website + $ asv --config contrib/asv.conf.json preview +''' + +from __future__ import absolute_import + +import functools +import os +import re + +from mercurial import ( + extensions, + hg, + ui as uimod, + util, +) + +basedir = os.path.abspath(os.path.join(os.path.dirname(__file__), + os.path.pardir, os.path.pardir)) +reposdir = os.environ['REPOS_DIR'] +reposnames = [name for name in os.listdir(reposdir) + if os.path.isdir(os.path.join(reposdir, name, ".hg"))] +if not reposnames: + raise ValueError("No repositories found in $REPO_DIR") +outputre = re.compile((r'! wall (\d+.\d+) comb \d+.\d+ user \d+.\d+ sys ' + r'\d+.\d+ \(best of \d+\)')) + +def runperfcommand(reponame, command, *args, **kwargs): + os.environ["HGRCPATH"] = os.environ.get("ASVHGRCPATH", "") + # for "historical portability" + # ui.load() has been available since d83ca85 + if util.safehasattr(uimod.ui, "load"): + ui = uimod.ui.load() + else: + ui = uimod.ui() + repo = hg.repository(ui, os.path.join(reposdir, reponame)) + perfext = extensions.load(ui, 'perfext', + os.path.join(basedir, 'contrib', 'perf.py')) + cmd = getattr(perfext, command) + ui.pushbuffer() + cmd(ui, repo, *args, **kwargs) + output = ui.popbuffer() + match = outputre.search(output) + if not match: + raise ValueError("Invalid output {0}".format(output)) + return float(match.group(1)) + +def perfbench(repos=reposnames, name=None, params=None): + """decorator to declare ASV benchmark based on contrib/perf.py extension + + An ASV benchmark is a python function with the given attributes: + + __name__: should start with track_, time_ or mem_ to be collected by ASV + params and param_name: parameter matrix to display multiple graphs on the + same page. + pretty_name: If defined it's displayed in web-ui instead of __name__ + (useful for revsets) + the module name is prepended to the benchmark name and displayed as + "category" in webui. + + Benchmarks are automatically parameterized with repositories found in the + REPOS_DIR environment variable. + + `params` is the param matrix in the form of a list of tuple + (param_name, [value0, value1]) + + For example [(x, [a, b]), (y, [c, d])] declare benchmarks for + (a, c), (a, d), (b, c) and (b, d). + """ + params = list(params or []) + params.insert(0, ("repo", repos)) + + def decorator(func): + @functools.wraps(func) + def wrapped(repo, *args): + def perf(command, *a, **kw): + return runperfcommand(repo, command, *a, **kw) + return func(perf, *args) + + wrapped.params = [p[1] for p in params] + wrapped.param_names = [p[0] for p in params] + wrapped.pretty_name = name + return wrapped + return decorator diff -Nru mercurial-4.0.2~trusty1/contrib/benchmarks/perf.py mercurial-4.1~rc~trusty1/contrib/benchmarks/perf.py --- mercurial-4.0.2~trusty1/contrib/benchmarks/perf.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/benchmarks/perf.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,26 @@ +# perf.py - asv benchmarks using contrib/perf.py extension +# +# Copyright 2016 Logilab SA +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +from . import perfbench + +@perfbench() +def track_tags(perf): + return perf("perftags") + +@perfbench() +def track_status(perf): + return perf("perfstatus", unknown=False) + +@perfbench(params=[('rev', ['1000', '10000', 'tip'])]) +def track_manifest(perf, rev): + return perf("perfmanifest", rev) + +@perfbench() +def track_heads(perf): + return perf("perfheads") diff -Nru mercurial-4.0.2~trusty1/contrib/benchmarks/revset.py mercurial-4.1~rc~trusty1/contrib/benchmarks/revset.py --- mercurial-4.0.2~trusty1/contrib/benchmarks/revset.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/benchmarks/revset.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,53 @@ +# revset.py - asv revset benchmarks +# +# Copyright 2016 Logilab SA +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +'''ASV revset benchmarks generated from contrib/base-revsets.txt + +Each revset benchmark is parameterized with variants (first, last, sort, ...) +''' + +from __future__ import absolute_import + +import os +import string +import sys + +from . import basedir, perfbench + +def createrevsetbenchmark(baseset, variants=None): + if variants is None: + # Default variants + variants = ["plain", "first", "last", "sort", "sort+first", + "sort+last"] + fname = "track_" + "_".join("".join([ + c if c in string.digits + string.letters else " " + for c in baseset + ]).split()) + + def wrap(fname, baseset): + @perfbench(name=baseset, params=[("variant", variants)]) + def f(perf, variant): + revset = baseset + if variant != "plain": + for var in variant.split("+"): + revset = "%s(%s)" % (var, revset) + return perf("perfrevset", revset) + f.__name__ = fname + return f + return wrap(fname, baseset) + +def initializerevsetbenchmarks(): + mod = sys.modules[__name__] + with open(os.path.join(basedir, 'contrib', 'base-revsets.txt'), + 'rb') as fh: + for line in fh: + baseset = line.strip() + if baseset and not baseset.startswith('#'): + func = createrevsetbenchmark(baseset) + setattr(mod, func.__name__, func) + +initializerevsetbenchmarks() diff -Nru mercurial-4.0.2~trusty1/contrib/check-code.py mercurial-4.1~rc~trusty1/contrib/check-code.py --- mercurial-4.0.2~trusty1/contrib/check-code.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/check-code.py 2017-01-18 16:54:51.000000000 +0000 @@ -142,7 +142,8 @@ (r'\|&', "don't use |&, use 2>&1"), (r'\w = +\w', "only one space after = allowed"), (r'\bsed\b.*[^\\]\\n', "don't use 'sed ... \\n', use a \\ and a newline"), - (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'") + (r'env.*-u', "don't use 'env -u VAR', use 'unset VAR'"), + (r'cp.* -r ', "don't use 'cp -r', use 'cp -R'"), ], # warnings [ @@ -324,6 +325,7 @@ # XXX only catch mutable arguments on the first line of the definition (r'def.*[( ]\w+=\{\}', "don't use mutable default arguments"), (r'\butil\.Abort\b', "directly use error.Abort"), + (r'^@(\w*\.)?cachefunc', "module-level @cachefunc is risky, please avoid"), (r'^import Queue', "don't use Queue, use util.queue + util.empty"), (r'^import cStringIO', "don't use cStringIO.StringIO, use util.stringio"), (r'^import urllib', "don't use urllib, use util.urlreq/util.urlerr"), @@ -455,8 +457,27 @@ [], ] +py3pats = [ + [ + (r'os\.environ', "use encoding.environ instead (py3)"), + (r'os\.name', "use pycompat.osname instead (py3)"), + (r'os\.getcwd', "use pycompat.getcwd instead (py3)"), + (r'os\.sep', "use pycompat.ossep instead (py3)"), + (r'os\.pathsep', "use pycompat.ospathsep instead (py3)"), + (r'os\.altsep', "use pycompat.osaltsep instead (py3)"), + (r'sys\.platform', "use pycompat.sysplatform instead (py3)"), + (r'getopt\.getopt', "use pycompat.getoptb instead (py3)"), + (r'os\.getenv', "use encoding.environ.get instead"), + (r'os\.setenv', "modifying the environ dict is not preferred"), + ], + # warnings + [], +] + checks = [ ('python', r'.*\.(py|cgi)$', r'^#!.*python', pyfilters, pypats), + ('python 3', r'.*(hgext|mercurial).*(? #include "hgclient.h" +#include "procutil.h" #include "util.h" -#ifndef UNIX_PATH_MAX -#define UNIX_PATH_MAX (sizeof(((struct sockaddr_un *)NULL)->sun_path)) +#ifndef PATH_MAX +#define PATH_MAX 4096 #endif struct cmdserveropts { - char sockname[UNIX_PATH_MAX]; - char redirectsockname[UNIX_PATH_MAX]; - char lockfile[UNIX_PATH_MAX]; + char sockname[PATH_MAX]; + char initsockname[PATH_MAX]; + char redirectsockname[PATH_MAX]; size_t argsize; const char **args; - int lockfd; - int sockdirfd; }; static void initcmdserveropts(struct cmdserveropts *opts) { memset(opts, 0, sizeof(struct cmdserveropts)); - opts->lockfd = -1; - opts->sockdirfd = -1; } static void freecmdserveropts(struct cmdserveropts *opts) { free(opts->args); opts->args = NULL; opts->argsize = 0; - assert(opts->lockfd == -1 && "should be closed by unlockcmdserver()"); - if (opts->sockdirfd >= 0) { - close(opts->sockdirfd); - opts->sockdirfd = -1; - } } /* @@ -136,68 +128,46 @@ abortmsg("insecure sockdir %s", sockdir); } -static void setcmdserveropts(struct cmdserveropts *opts) +static void getdefaultsockdir(char sockdir[], size_t size) { + /* by default, put socket file in secure directory + * (${XDG_RUNTIME_DIR}/chg, or /${TMPDIR:-tmp}/chg$UID) + * (permission of socket file may be ignored on some Unices) */ + const char *runtimedir = getenv("XDG_RUNTIME_DIR"); int r; - char sockdir[UNIX_PATH_MAX]; - const char *envsockname = getenv("CHGSOCKNAME"); - if (!envsockname) { - /* by default, put socket file in secure directory - * (permission of socket file may be ignored on some Unices) */ + if (runtimedir) { + r = snprintf(sockdir, size, "%s/chg", runtimedir); + } else { const char *tmpdir = getenv("TMPDIR"); if (!tmpdir) tmpdir = "/tmp"; - r = snprintf(sockdir, sizeof(sockdir), "%s/chg%d", - tmpdir, geteuid()); - if (r < 0 || (size_t)r >= sizeof(sockdir)) - abortmsg("too long TMPDIR (r = %d)", r); + r = snprintf(sockdir, size, "%s/chg%d", tmpdir, geteuid()); + } + if (r < 0 || (size_t)r >= size) + abortmsg("too long TMPDIR (r = %d)", r); +} + +static void setcmdserveropts(struct cmdserveropts *opts) +{ + int r; + char sockdir[PATH_MAX]; + const char *envsockname = getenv("CHGSOCKNAME"); + if (!envsockname) { + getdefaultsockdir(sockdir, sizeof(sockdir)); preparesockdir(sockdir); } const char *basename = (envsockname) ? envsockname : sockdir; const char *sockfmt = (envsockname) ? "%s" : "%s/server"; - const char *lockfmt = (envsockname) ? "%s.lock" : "%s/lock"; r = snprintf(opts->sockname, sizeof(opts->sockname), sockfmt, basename); if (r < 0 || (size_t)r >= sizeof(opts->sockname)) abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r); - r = snprintf(opts->lockfile, sizeof(opts->lockfile), lockfmt, basename); - if (r < 0 || (size_t)r >= sizeof(opts->lockfile)) + r = snprintf(opts->initsockname, sizeof(opts->initsockname), + "%s.%u", opts->sockname, (unsigned)getpid()); + if (r < 0 || (size_t)r >= sizeof(opts->initsockname)) abortmsg("too long TMPDIR or CHGSOCKNAME (r = %d)", r); } -/* - * Acquire a file lock that indicates a client is trying to start and connect - * to a server, before executing a command. The lock is released upon exit or - * explicit unlock. Will block if the lock is held by another process. - */ -static void lockcmdserver(struct cmdserveropts *opts) -{ - if (opts->lockfd == -1) { - opts->lockfd = open(opts->lockfile, - O_RDWR | O_CREAT | O_NOFOLLOW, 0600); - if (opts->lockfd == -1) - abortmsgerrno("cannot create lock file %s", - opts->lockfile); - fsetcloexec(opts->lockfd); - } - int r = flock(opts->lockfd, LOCK_EX); - if (r == -1) - abortmsgerrno("cannot acquire lock"); -} - -/* - * Release the file lock held by calling lockcmdserver. Will do nothing if - * lockcmdserver is not called. - */ -static void unlockcmdserver(struct cmdserveropts *opts) -{ - if (opts->lockfd == -1) - return; - flock(opts->lockfd, LOCK_UN); - close(opts->lockfd); - opts->lockfd = -1; -} - static const char *gethgcmd(void) { static const char *hgcmd = NULL; @@ -223,9 +193,8 @@ hgcmd, "serve", "--cmdserver", "chgunix", - "--address", opts->sockname, + "--address", opts->initsockname, "--daemon-postexec", "chdir:/", - "--config", "extensions.chgserver=", }; size_t baseargvsize = sizeof(baseargv) / sizeof(baseargv[0]); size_t argsize = baseargvsize + opts->argsize + 1; @@ -248,7 +217,7 @@ static const struct timespec sleepreq = {0, 10 * 1000000}; int pst = 0; - debugmsg("try connect to %s repeatedly", opts->sockname); + debugmsg("try connect to %s repeatedly", opts->initsockname); unsigned int timeoutsec = 60; /* default: 60 seconds */ const char *timeoutenv = getenv("CHGTIMEOUT"); @@ -256,9 +225,15 @@ sscanf(timeoutenv, "%u", &timeoutsec); for (unsigned int i = 0; !timeoutsec || i < timeoutsec * 100; i++) { - hgclient_t *hgc = hgc_open(opts->sockname); - if (hgc) + hgclient_t *hgc = hgc_open(opts->initsockname); + if (hgc) { + debugmsg("rename %s to %s", opts->initsockname, + opts->sockname); + int r = rename(opts->initsockname, opts->sockname); + if (r != 0) + abortmsgerrno("cannot rename"); return hgc; + } if (pid > 0) { /* collect zombie if child process fails to start */ @@ -270,7 +245,7 @@ nanosleep(&sleepreq, NULL); } - abortmsg("timed out waiting for cmdserver %s", opts->sockname); + abortmsg("timed out waiting for cmdserver %s", opts->initsockname); return NULL; cleanup: @@ -298,14 +273,6 @@ if (hgc) return hgc; - lockcmdserver(opts); - hgc = hgc_open(sockname); - if (hgc) { - unlockcmdserver(opts); - debugmsg("cmdserver is started by another process"); - return hgc; - } - /* prevent us from being connected to an outdated server: we were * told by a server to redirect to opts->redirectsockname and that * address does not work. we do not want to connect to the server @@ -313,7 +280,7 @@ if (sockname == opts->redirectsockname) unlink(opts->sockname); - debugmsg("start cmdserver at %s", opts->sockname); + debugmsg("start cmdserver at %s", opts->initsockname); pid_t pid = fork(); if (pid < 0) @@ -324,7 +291,6 @@ hgc = retryconnectcmdserver(opts, pid); } - unlockcmdserver(opts); return hgc; } @@ -338,215 +304,6 @@ } } -static pid_t pagerpid = 0; -static pid_t peerpgid = 0; -static pid_t peerpid = 0; - -static void forwardsignal(int sig) -{ - assert(peerpid > 0); - if (kill(peerpid, sig) < 0) - abortmsgerrno("cannot kill %d", peerpid); - debugmsg("forward signal %d", sig); -} - -static void forwardsignaltogroup(int sig) -{ - /* prefer kill(-pgid, sig), fallback to pid if pgid is invalid */ - pid_t killpid = peerpgid > 1 ? -peerpgid : peerpid; - if (kill(killpid, sig) < 0) - abortmsgerrno("cannot kill %d", killpid); - debugmsg("forward signal %d to %d", sig, killpid); -} - -static void handlestopsignal(int sig) -{ - sigset_t unblockset, oldset; - struct sigaction sa, oldsa; - if (sigemptyset(&unblockset) < 0) - goto error; - if (sigaddset(&unblockset, sig) < 0) - goto error; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = SIG_DFL; - sa.sa_flags = SA_RESTART; - if (sigemptyset(&sa.sa_mask) < 0) - goto error; - - forwardsignal(sig); - if (raise(sig) < 0) /* resend to self */ - goto error; - if (sigaction(sig, &sa, &oldsa) < 0) - goto error; - if (sigprocmask(SIG_UNBLOCK, &unblockset, &oldset) < 0) - goto error; - /* resent signal will be handled before sigprocmask() returns */ - if (sigprocmask(SIG_SETMASK, &oldset, NULL) < 0) - goto error; - if (sigaction(sig, &oldsa, NULL) < 0) - goto error; - return; - -error: - abortmsgerrno("failed to handle stop signal"); -} - -static void handlechildsignal(int sig UNUSED_) -{ - if (peerpid == 0 || pagerpid == 0) - return; - /* if pager exits, notify the server with SIGPIPE immediately. - * otherwise the server won't get SIGPIPE if it does not write - * anything. (issue5278) */ - if (waitpid(pagerpid, NULL, WNOHANG) == pagerpid) - kill(peerpid, SIGPIPE); -} - -static void setupsignalhandler(const hgclient_t *hgc) -{ - pid_t pid = hgc_peerpid(hgc); - if (pid <= 0) - return; - peerpid = pid; - - pid_t pgid = hgc_peerpgid(hgc); - peerpgid = (pgid <= 1 ? 0 : pgid); - - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = forwardsignaltogroup; - sa.sa_flags = SA_RESTART; - if (sigemptyset(&sa.sa_mask) < 0) - goto error; - - if (sigaction(SIGHUP, &sa, NULL) < 0) - goto error; - if (sigaction(SIGINT, &sa, NULL) < 0) - goto error; - - /* terminate frontend by double SIGTERM in case of server freeze */ - sa.sa_handler = forwardsignal; - sa.sa_flags |= SA_RESETHAND; - if (sigaction(SIGTERM, &sa, NULL) < 0) - goto error; - - /* notify the worker about window resize events */ - sa.sa_flags = SA_RESTART; - if (sigaction(SIGWINCH, &sa, NULL) < 0) - goto error; - /* propagate job control requests to worker */ - sa.sa_handler = forwardsignal; - sa.sa_flags = SA_RESTART; - if (sigaction(SIGCONT, &sa, NULL) < 0) - goto error; - sa.sa_handler = handlestopsignal; - sa.sa_flags = SA_RESTART; - if (sigaction(SIGTSTP, &sa, NULL) < 0) - goto error; - /* get notified when pager exits */ - sa.sa_handler = handlechildsignal; - sa.sa_flags = SA_RESTART; - if (sigaction(SIGCHLD, &sa, NULL) < 0) - goto error; - - return; - -error: - abortmsgerrno("failed to set up signal handlers"); -} - -static void restoresignalhandler() -{ - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_handler = SIG_DFL; - sa.sa_flags = SA_RESTART; - if (sigemptyset(&sa.sa_mask) < 0) - goto error; - - if (sigaction(SIGHUP, &sa, NULL) < 0) - goto error; - if (sigaction(SIGTERM, &sa, NULL) < 0) - goto error; - if (sigaction(SIGWINCH, &sa, NULL) < 0) - goto error; - if (sigaction(SIGCONT, &sa, NULL) < 0) - goto error; - if (sigaction(SIGTSTP, &sa, NULL) < 0) - goto error; - if (sigaction(SIGCHLD, &sa, NULL) < 0) - goto error; - - /* ignore Ctrl+C while shutting down to make pager exits cleanly */ - sa.sa_handler = SIG_IGN; - if (sigaction(SIGINT, &sa, NULL) < 0) - goto error; - - peerpid = 0; - return; - -error: - abortmsgerrno("failed to restore signal handlers"); -} - -/* This implementation is based on hgext/pager.py (post 369741ef7253) - * Return 0 if pager is not started, or pid of the pager */ -static pid_t setuppager(hgclient_t *hgc, const char *const args[], - size_t argsize) -{ - const char *pagercmd = hgc_getpager(hgc, args, argsize); - if (!pagercmd) - return 0; - - int pipefds[2]; - if (pipe(pipefds) < 0) - return 0; - pid_t pid = fork(); - if (pid < 0) - goto error; - if (pid > 0) { - close(pipefds[0]); - if (dup2(pipefds[1], fileno(stdout)) < 0) - goto error; - if (isatty(fileno(stderr))) { - if (dup2(pipefds[1], fileno(stderr)) < 0) - goto error; - } - close(pipefds[1]); - hgc_attachio(hgc); /* reattach to pager */ - return pid; - } else { - dup2(pipefds[0], fileno(stdin)); - close(pipefds[0]); - close(pipefds[1]); - - int r = execlp("/bin/sh", "/bin/sh", "-c", pagercmd, NULL); - if (r < 0) { - abortmsgerrno("cannot start pager '%s'", pagercmd); - } - return 0; - } - -error: - close(pipefds[0]); - close(pipefds[1]); - abortmsgerrno("failed to prepare pager"); - return 0; -} - -static void waitpager(pid_t pid) -{ - /* close output streams to notify the pager its input ends */ - fclose(stdout); - fclose(stderr); - while (1) { - pid_t ret = waitpid(pid, NULL, 0); - if (ret == -1 && errno == EINTR) - continue; - break; - } -} - /* Run instructions sent from the server like unlink and set redirect path * Return 1 if reconnect is needed, otherwise 0 */ static int runinstructions(struct cmdserveropts *opts, const char **insts) @@ -671,14 +428,12 @@ gethgcmd()); } - setupsignalhandler(hgc); - pagerpid = setuppager(hgc, argv + 1, argc - 1); + setupsignalhandler(hgc_peerpid(hgc), hgc_peerpgid(hgc)); int exitcode = hgc_runcommand(hgc, argv + 1, argc - 1); restoresignalhandler(); hgc_close(hgc); freecmdserveropts(&opts); - if (pagerpid) - waitpager(pagerpid); + waitpager(); return exitcode; } diff -Nru mercurial-4.0.2~trusty1/contrib/chg/hgclient.c mercurial-4.1~rc~trusty1/contrib/chg/hgclient.c --- mercurial-4.0.2~trusty1/contrib/chg/hgclient.c 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/chg/hgclient.c 2017-01-18 16:54:51.000000000 +0000 @@ -23,6 +23,7 @@ #include #include "hgclient.h" +#include "procutil.h" #include "util.h" enum { @@ -31,10 +32,10 @@ /* cHg extension: */ CAP_ATTACHIO = 0x0100, CAP_CHDIR = 0x0200, - CAP_GETPAGER = 0x0400, CAP_SETENV = 0x0800, CAP_SETUMASK = 0x1000, CAP_VALIDATE = 0x2000, + CAP_SETPROCNAME = 0x4000, }; typedef struct { @@ -47,10 +48,10 @@ {"runcommand", CAP_RUNCOMMAND}, {"attachio", CAP_ATTACHIO}, {"chdir", CAP_CHDIR}, - {"getpager", CAP_GETPAGER}, {"setenv", CAP_SETENV}, {"setumask", CAP_SETUMASK}, {"validate", CAP_VALIDATE}, + {"setprocname", CAP_SETPROCNAME}, {NULL, 0}, /* terminator */ }; @@ -71,6 +72,8 @@ static const size_t defaultdatasize = 4096; +static void attachio(hgclient_t *hgc); + static void initcontext(context_t *ctx) { ctx->ch = '\0'; @@ -237,16 +240,27 @@ ctx->data[ctx->datasize] = '\0'; /* terminate last string */ const char **args = unpackcmdargsnul(ctx); - if (!args[0] || !args[1]) - abortmsg("missing command or cwd in system request"); - debugmsg("run '%s' at '%s'", args[0], args[1]); - int32_t r = runshellcmd(args[0], args + 2, args[1]); - free(args); - - uint32_t r_n = htonl(r); - memcpy(ctx->data, &r_n, sizeof(r_n)); - ctx->datasize = sizeof(r_n); - writeblock(hgc); + if (!args[0] || !args[1] || !args[2]) + abortmsg("missing type or command or cwd in system request"); + if (strcmp(args[0], "system") == 0) { + debugmsg("run '%s' at '%s'", args[1], args[2]); + int32_t r = runshellcmd(args[1], args + 3, args[2]); + free(args); + + uint32_t r_n = htonl(r); + memcpy(ctx->data, &r_n, sizeof(r_n)); + ctx->datasize = sizeof(r_n); + writeblock(hgc); + } else if (strcmp(args[0], "pager") == 0) { + setuppager(args[1]); + if (hgc->capflags & CAP_ATTACHIO) + attachio(hgc); + /* unblock the server */ + static const char emptycmd[] = "\n"; + sendall(hgc->sockfd, emptycmd, sizeof(emptycmd) - 1); + } else { + abortmsg("unknown type in system request: %s", args[0]); + } } /* Read response of command execution until receiving 'r'-esult */ @@ -350,6 +364,16 @@ debugmsg("capflags=0x%04x, pid=%d", hgc->capflags, hgc->pid); } +static void updateprocname(hgclient_t *hgc) +{ + int r = snprintf(hgc->ctx.data, hgc->ctx.maxdatasize, + "chg[worker/%d]", (int)getpid()); + if (r < 0 || (size_t)r >= hgc->ctx.maxdatasize) + abortmsg("insufficient buffer to write procname (r = %d)", r); + hgc->ctx.datasize = (size_t)r; + writeblockrequest(hgc, "setprocname"); +} + static void attachio(hgclient_t *hgc) { debugmsg("request attachio"); @@ -425,15 +449,49 @@ struct sockaddr_un addr; addr.sun_family = AF_UNIX; - strncpy(addr.sun_path, sockname, sizeof(addr.sun_path)); + + /* use chdir to workaround small sizeof(sun_path) */ + int bakfd = -1; + const char *basename = sockname; + { + const char *split = strrchr(sockname, '/'); + if (split && split != sockname) { + if (split[1] == '\0') + abortmsg("sockname cannot end with a slash"); + size_t len = split - sockname; + char sockdir[len + 1]; + memcpy(sockdir, sockname, len); + sockdir[len] = '\0'; + + bakfd = open(".", O_DIRECTORY); + if (bakfd == -1) + abortmsgerrno("cannot open cwd"); + + int r = chdir(sockdir); + if (r != 0) + abortmsgerrno("cannot chdir %s", sockdir); + + basename = split + 1; + } + } + if (strlen(basename) >= sizeof(addr.sun_path)) + abortmsg("sockname is too long: %s", basename); + strncpy(addr.sun_path, basename, sizeof(addr.sun_path)); addr.sun_path[sizeof(addr.sun_path) - 1] = '\0'; + /* real connect */ int r = connect(fd, (struct sockaddr *)&addr, sizeof(addr)); if (r < 0) { + if (errno != ENOENT && errno != ECONNREFUSED) + abortmsgerrno("cannot connect to %s", sockname); + } + if (bakfd != -1) { + fchdirx(bakfd); + close(bakfd); + } + if (r < 0) { close(fd); - if (errno == ENOENT || errno == ECONNREFUSED) - return NULL; - abortmsgerrno("cannot connect to %s", addr.sun_path); + return NULL; } debugmsg("connected to %s", addr.sun_path); @@ -445,6 +503,8 @@ readhello(hgc); if (!(hgc->capflags & CAP_RUNCOMMAND)) abortmsg("insufficient capability: runcommand"); + if (hgc->capflags & CAP_SETPROCNAME) + updateprocname(hgc); if (hgc->capflags & CAP_ATTACHIO) attachio(hgc); if (hgc->capflags & CAP_CHDIR) @@ -545,31 +605,6 @@ } /*! - * Get pager command for the given Mercurial command args - * - * If no pager enabled, returns NULL. The return value becomes invalid - * once you run another request to hgc. - */ -const char *hgc_getpager(hgclient_t *hgc, const char *const args[], - size_t argsize) -{ - assert(hgc); - - if (!(hgc->capflags & CAP_GETPAGER)) - return NULL; - - packcmdargs(&hgc->ctx, args, argsize); - writeblockrequest(hgc, "getpager"); - handleresponse(hgc); - - if (hgc->ctx.datasize < 1 || hgc->ctx.data[0] == '\0') - return NULL; - enlargecontext(&hgc->ctx, hgc->ctx.datasize + 1); - hgc->ctx.data[hgc->ctx.datasize] = '\0'; - return hgc->ctx.data; -} - -/*! * Update server's environment variables * * @param envp list of environment variables in "NAME=VALUE" format, diff -Nru mercurial-4.0.2~trusty1/contrib/chg/hgclient.h mercurial-4.1~rc~trusty1/contrib/chg/hgclient.h --- mercurial-4.0.2~trusty1/contrib/chg/hgclient.h 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/chg/hgclient.h 2017-01-18 16:54:51.000000000 +0000 @@ -25,8 +25,6 @@ size_t argsize); int hgc_runcommand(hgclient_t *hgc, const char *const args[], size_t argsize); void hgc_attachio(hgclient_t *hgc); -const char *hgc_getpager(hgclient_t *hgc, const char *const args[], - size_t argsize); void hgc_setenv(hgclient_t *hgc, const char *const envp[]); #endif /* HGCLIENT_H_ */ diff -Nru mercurial-4.0.2~trusty1/contrib/chg/Makefile mercurial-4.1~rc~trusty1/contrib/chg/Makefile --- mercurial-4.0.2~trusty1/contrib/chg/Makefile 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/chg/Makefile 2017-01-18 16:54:51.000000000 +0000 @@ -1,7 +1,7 @@ HG = $(CURDIR)/../../hg TARGET = chg -SRCS = chg.c hgclient.c util.c +SRCS = chg.c hgclient.c procutil.c util.c OBJS = $(SRCS:.c=.o) CFLAGS ?= -O2 -Wall -Wextra -pedantic -g @@ -24,8 +24,9 @@ $(TARGET): $(OBJS) $(CC) $(LDFLAGS) -o $@ $(OBJS) -chg.o: hgclient.h util.h -hgclient.o: hgclient.h util.h +chg.o: hgclient.h procutil.h util.h +hgclient.o: hgclient.h procutil.h util.h +procutil.o: procutil.h util.h util.o: util.h .PHONY: install @@ -40,7 +41,6 @@ [ -d $(CHGSOCKDIR) ] || ( umask 077; mkdir $(CHGSOCKDIR) ) $(HG) serve --cwd / --cmdserver chgunix \ --address $(CHGSOCKNAME) \ - --config extensions.chgserver= \ --config cmdserver.log=/dev/stderr .PHONY: clean diff -Nru mercurial-4.0.2~trusty1/contrib/chg/procutil.c mercurial-4.1~rc~trusty1/contrib/chg/procutil.c --- mercurial-4.0.2~trusty1/contrib/chg/procutil.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/chg/procutil.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,227 @@ +/* + * Utilities about process handling - signal and subprocess (ex. pager) + * + * Copyright (c) 2011 Yuya Nishihara + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2 or any later version. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "procutil.h" +#include "util.h" + +static pid_t pagerpid = 0; +static pid_t peerpgid = 0; +static pid_t peerpid = 0; + +static void forwardsignal(int sig) +{ + assert(peerpid > 0); + if (kill(peerpid, sig) < 0) + abortmsgerrno("cannot kill %d", peerpid); + debugmsg("forward signal %d", sig); +} + +static void forwardsignaltogroup(int sig) +{ + /* prefer kill(-pgid, sig), fallback to pid if pgid is invalid */ + pid_t killpid = peerpgid > 1 ? -peerpgid : peerpid; + if (kill(killpid, sig) < 0) + abortmsgerrno("cannot kill %d", killpid); + debugmsg("forward signal %d to %d", sig, killpid); +} + +static void handlestopsignal(int sig) +{ + sigset_t unblockset, oldset; + struct sigaction sa, oldsa; + if (sigemptyset(&unblockset) < 0) + goto error; + if (sigaddset(&unblockset, sig) < 0) + goto error; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = SIG_DFL; + sa.sa_flags = SA_RESTART; + if (sigemptyset(&sa.sa_mask) < 0) + goto error; + + forwardsignal(sig); + if (raise(sig) < 0) /* resend to self */ + goto error; + if (sigaction(sig, &sa, &oldsa) < 0) + goto error; + if (sigprocmask(SIG_UNBLOCK, &unblockset, &oldset) < 0) + goto error; + /* resent signal will be handled before sigprocmask() returns */ + if (sigprocmask(SIG_SETMASK, &oldset, NULL) < 0) + goto error; + if (sigaction(sig, &oldsa, NULL) < 0) + goto error; + return; + +error: + abortmsgerrno("failed to handle stop signal"); +} + +static void handlechildsignal(int sig UNUSED_) +{ + if (peerpid == 0 || pagerpid == 0) + return; + /* if pager exits, notify the server with SIGPIPE immediately. + * otherwise the server won't get SIGPIPE if it does not write + * anything. (issue5278) */ + if (waitpid(pagerpid, NULL, WNOHANG) == pagerpid) + kill(peerpid, SIGPIPE); +} + +void setupsignalhandler(pid_t pid, pid_t pgid) +{ + if (pid <= 0) + return; + peerpid = pid; + peerpgid = (pgid <= 1 ? 0 : pgid); + + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = forwardsignaltogroup; + sa.sa_flags = SA_RESTART; + if (sigemptyset(&sa.sa_mask) < 0) + goto error; + + if (sigaction(SIGHUP, &sa, NULL) < 0) + goto error; + if (sigaction(SIGINT, &sa, NULL) < 0) + goto error; + + /* terminate frontend by double SIGTERM in case of server freeze */ + sa.sa_handler = forwardsignal; + sa.sa_flags |= SA_RESETHAND; + if (sigaction(SIGTERM, &sa, NULL) < 0) + goto error; + + /* notify the worker about window resize events */ + sa.sa_flags = SA_RESTART; + if (sigaction(SIGWINCH, &sa, NULL) < 0) + goto error; + /* propagate job control requests to worker */ + sa.sa_handler = forwardsignal; + sa.sa_flags = SA_RESTART; + if (sigaction(SIGCONT, &sa, NULL) < 0) + goto error; + sa.sa_handler = handlestopsignal; + sa.sa_flags = SA_RESTART; + if (sigaction(SIGTSTP, &sa, NULL) < 0) + goto error; + /* get notified when pager exits */ + sa.sa_handler = handlechildsignal; + sa.sa_flags = SA_RESTART; + if (sigaction(SIGCHLD, &sa, NULL) < 0) + goto error; + + return; + +error: + abortmsgerrno("failed to set up signal handlers"); +} + +void restoresignalhandler(void) +{ + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_handler = SIG_DFL; + sa.sa_flags = SA_RESTART; + if (sigemptyset(&sa.sa_mask) < 0) + goto error; + + if (sigaction(SIGHUP, &sa, NULL) < 0) + goto error; + if (sigaction(SIGTERM, &sa, NULL) < 0) + goto error; + if (sigaction(SIGWINCH, &sa, NULL) < 0) + goto error; + if (sigaction(SIGCONT, &sa, NULL) < 0) + goto error; + if (sigaction(SIGTSTP, &sa, NULL) < 0) + goto error; + if (sigaction(SIGCHLD, &sa, NULL) < 0) + goto error; + + /* ignore Ctrl+C while shutting down to make pager exits cleanly */ + sa.sa_handler = SIG_IGN; + if (sigaction(SIGINT, &sa, NULL) < 0) + goto error; + + peerpid = 0; + return; + +error: + abortmsgerrno("failed to restore signal handlers"); +} + +/* This implementation is based on hgext/pager.py (post 369741ef7253) + * Return 0 if pager is not started, or pid of the pager */ +pid_t setuppager(const char *pagercmd) +{ + assert(pagerpid == 0); + if (!pagercmd) + return 0; + + int pipefds[2]; + if (pipe(pipefds) < 0) + return 0; + pid_t pid = fork(); + if (pid < 0) + goto error; + if (pid > 0) { + close(pipefds[0]); + if (dup2(pipefds[1], fileno(stdout)) < 0) + goto error; + if (isatty(fileno(stderr))) { + if (dup2(pipefds[1], fileno(stderr)) < 0) + goto error; + } + close(pipefds[1]); + pagerpid = pid; + return pid; + } else { + dup2(pipefds[0], fileno(stdin)); + close(pipefds[0]); + close(pipefds[1]); + + int r = execlp("/bin/sh", "/bin/sh", "-c", pagercmd, NULL); + if (r < 0) { + abortmsgerrno("cannot start pager '%s'", pagercmd); + } + return 0; + } + +error: + close(pipefds[0]); + close(pipefds[1]); + abortmsgerrno("failed to prepare pager"); + return 0; +} + +void waitpager(void) +{ + if (pagerpid == 0) + return; + + /* close output streams to notify the pager its input ends */ + fclose(stdout); + fclose(stderr); + while (1) { + pid_t ret = waitpid(pagerpid, NULL, 0); + if (ret == -1 && errno == EINTR) + continue; + break; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/chg/procutil.h mercurial-4.1~rc~trusty1/contrib/chg/procutil.h --- mercurial-4.0.2~trusty1/contrib/chg/procutil.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/chg/procutil.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,21 @@ +/* + * Utilities about process handling - signal and subprocess (ex. pager) + * + * Copyright (c) 2011 Yuya Nishihara + * + * This software may be used and distributed according to the terms of the + * GNU General Public License version 2 or any later version. + */ + +#ifndef PROCUTIL_H_ +#define PROCUTIL_H_ + +#include + +void restoresignalhandler(void); +void setupsignalhandler(pid_t pid, pid_t pgid); + +pid_t setuppager(const char *pagercmd); +void waitpager(void); + +#endif /* PROCUTIL_H_ */ diff -Nru mercurial-4.0.2~trusty1/contrib/debugshell.py mercurial-4.1~rc~trusty1/contrib/debugshell.py --- mercurial-4.0.2~trusty1/contrib/debugshell.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/debugshell.py 2017-01-18 16:54:51.000000000 +0000 @@ -18,7 +18,7 @@ 'mercurial': mercurial, 'repo': repo, 'cl': repo.changelog, - 'mf': repo.manifest, + 'mf': repo.manifestlog, } code.interact(msg, local=objects) @@ -27,7 +27,7 @@ import IPython cl = repo.changelog - mf = repo.manifest + mf = repo.manifestlog cl, mf # use variables to appease pyflakes IPython.embed() diff -Nru mercurial-4.0.2~trusty1/contrib/docker/ubuntu-yakkety mercurial-4.1~rc~trusty1/contrib/docker/ubuntu-yakkety --- mercurial-4.0.2~trusty1/contrib/docker/ubuntu-yakkety 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/docker/ubuntu-yakkety 2017-01-19 01:13:13.000000000 +0000 @@ -0,0 +1,11 @@ +FROM ubuntu:yakkety +RUN apt-get update && apt-get install -y \ + build-essential \ + debhelper \ + dh-python \ + devscripts \ + python \ + python-all-dev \ + python-docutils \ + zip \ + unzip diff -Nru mercurial-4.0.2~trusty1/contrib/import-checker.py mercurial-4.1~rc~trusty1/contrib/import-checker.py --- mercurial-4.0.2~trusty1/contrib/import-checker.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/import-checker.py 2017-01-18 16:54:51.000000000 +0000 @@ -391,6 +391,7 @@ seennonsymbollocal = False # The last name to be imported (for sorting). lastname = None + laststdlib = None # Relative import levels encountered so far. seenlevels = set() @@ -412,16 +413,18 @@ name = node.names[0].name asname = node.names[0].asname + stdlib = name in stdlib_modules + # Ignore sorting rules on imports inside blocks. if node.col_offset == root_col_offset: - if lastname and name < lastname: + if lastname and name < lastname and laststdlib == stdlib: yield msg('imports not lexically sorted: %s < %s', name, lastname) - lastname = name + lastname = name + laststdlib = stdlib # stdlib imports should be before local imports. - stdlib = name in stdlib_modules if stdlib and seenlocal and node.col_offset == root_col_offset: yield msg('stdlib import "%s" follows local import: %s', name, seenlocal) diff -Nru mercurial-4.0.2~trusty1/contrib/memory.py mercurial-4.1~rc~trusty1/contrib/memory.py --- mercurial-4.0.2~trusty1/contrib/memory.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/memory.py 2017-01-18 16:54:51.000000000 +0000 @@ -25,8 +25,8 @@ key = parts[0][2:-1].lower() if key in result: result[key] = int(parts[1]) - ui.write_err(", ".join(["%s: %.1f MiB" % (key, value / 1024.0) - for key, value in result.iteritems()]) + "\n") + ui.write_err(", ".join(["%s: %.1f MiB" % (k, v / 1024.0) + for k, v in result.iteritems()]) + "\n") def extsetup(ui): atexit.register(memusage, ui) diff -Nru mercurial-4.0.2~trusty1/contrib/perf.py mercurial-4.1~rc~trusty1/contrib/perf.py --- mercurial-4.0.2~trusty1/contrib/perf.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/perf.py 2017-01-18 16:54:51.000000000 +0000 @@ -25,6 +25,7 @@ import sys import time from mercurial import ( + bdiff, changegroup, cmdutil, commands, @@ -33,7 +34,6 @@ extensions, mdiff, merge, - revlog, util, ) @@ -135,13 +135,14 @@ if opts is None: opts = {} - # redirect all to stderr - ui = ui.copy() - uifout = safeattrsetter(ui, 'fout', ignoremissing=True) - if uifout: - # for "historical portability": - # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d) - uifout.set(ui.ferr) + # redirect all to stderr unless buffer api is in use + if not ui._buffers: + ui = ui.copy() + uifout = safeattrsetter(ui, 'fout', ignoremissing=True) + if uifout: + # for "historical portability": + # ui.fout/ferr have been available since 1.9 (or 4e1ccd4c2b6d) + uifout.set(ui.ferr) # get a formatter uiformatter = getattr(ui, 'formatter', None) @@ -565,8 +566,8 @@ ctx = scmutil.revsingle(repo, rev, rev) t = ctx.manifestnode() def d(): - repo.manifest.clearcaches() - repo.manifest.read(t) + repo.manifestlog.clearcaches() + repo.manifestlog[t].read() timer(d) fm.end() @@ -746,6 +747,64 @@ timer(d) fm.end() +@command('perfbdiff', revlogopts + formatteropts + [ + ('', 'count', 1, 'number of revisions to test (when using --startrev)'), + ('', 'alldata', False, 'test bdiffs for all associated revisions')], + '-c|-m|FILE REV') +def perfbdiff(ui, repo, file_, rev=None, count=None, **opts): + """benchmark a bdiff between revisions + + By default, benchmark a bdiff between its delta parent and itself. + + With ``--count``, benchmark bdiffs between delta parents and self for N + revisions starting at the specified revision. + + With ``--alldata``, assume the requested revision is a changeset and + measure bdiffs for all changes related to that changeset (manifest + and filelogs). + """ + if opts['alldata']: + opts['changelog'] = True + + if opts.get('changelog') or opts.get('manifest'): + file_, rev = None, file_ + elif rev is None: + raise error.CommandError('perfbdiff', 'invalid arguments') + + textpairs = [] + + r = cmdutil.openrevlog(repo, 'perfbdiff', file_, opts) + + startrev = r.rev(r.lookup(rev)) + for rev in range(startrev, min(startrev + count, len(r) - 1)): + if opts['alldata']: + # Load revisions associated with changeset. + ctx = repo[rev] + mtext = repo.manifestlog._revlog.revision(ctx.manifestnode()) + for pctx in ctx.parents(): + pman = repo.manifestlog._revlog.revision(pctx.manifestnode()) + textpairs.append((pman, mtext)) + + # Load filelog revisions by iterating manifest delta. + man = ctx.manifest() + pman = ctx.p1().manifest() + for filename, change in pman.diff(man).items(): + fctx = repo.file(filename) + f1 = fctx.revision(change[0][0] or -1) + f2 = fctx.revision(change[1][0] or -1) + textpairs.append((f1, f2)) + else: + dp = r.deltaparent(rev) + textpairs.append((r.revision(dp), r.revision(rev))) + + def d(): + for pair in textpairs: + bdiff.bdiff(*pair) + + timer, fm = gettimer(ui, opts) + timer(d) + fm.end() + @command('perfdiffwd', formatteropts) def perfdiffwd(ui, repo, **opts): """Profile diff of working directory changes""" @@ -799,6 +858,116 @@ timer(d) fm.end() +@command('perfrevlogchunks', revlogopts + formatteropts + + [('e', 'engines', '', 'compression engines to use'), + ('s', 'startrev', 0, 'revision to start at')], + '-c|-m|FILE') +def perfrevlogchunks(ui, repo, file_=None, engines=None, startrev=0, **opts): + """Benchmark operations on revlog chunks. + + Logically, each revlog is a collection of fulltext revisions. However, + stored within each revlog are "chunks" of possibly compressed data. This + data needs to be read and decompressed or compressed and written. + + This command measures the time it takes to read+decompress and recompress + chunks in a revlog. It effectively isolates I/O and compression performance. + For measurements of higher-level operations like resolving revisions, + see ``perfrevlog`` and ``perfrevlogrevision``. + """ + rl = cmdutil.openrevlog(repo, 'perfrevlogchunks', file_, opts) + + # Verify engines argument. + if engines: + engines = set(e.strip() for e in engines.split(',')) + for engine in engines: + try: + util.compressionengines[engine] + except KeyError: + raise error.Abort('unknown compression engine: %s' % engine) + else: + engines = [] + for e in util.compengines: + engine = util.compengines[e] + try: + if engine.available(): + engine.revlogcompressor().compress('dummy') + engines.append(e) + except NotImplementedError: + pass + + revs = list(rl.revs(startrev, len(rl) - 1)) + + def rlfh(rl): + if rl._inline: + return getsvfs(repo)(rl.indexfile) + else: + return getsvfs(repo)(rl.datafile) + + def doread(): + rl.clearcaches() + for rev in revs: + rl._chunkraw(rev, rev) + + def doreadcachedfh(): + rl.clearcaches() + fh = rlfh(rl) + for rev in revs: + rl._chunkraw(rev, rev, df=fh) + + def doreadbatch(): + rl.clearcaches() + rl._chunkraw(revs[0], revs[-1]) + + def doreadbatchcachedfh(): + rl.clearcaches() + fh = rlfh(rl) + rl._chunkraw(revs[0], revs[-1], df=fh) + + def dochunk(): + rl.clearcaches() + fh = rlfh(rl) + for rev in revs: + rl._chunk(rev, df=fh) + + chunks = [None] + + def dochunkbatch(): + rl.clearcaches() + fh = rlfh(rl) + # Save chunks as a side-effect. + chunks[0] = rl._chunks(revs, df=fh) + + def docompress(compressor): + rl.clearcaches() + + try: + # Swap in the requested compression engine. + oldcompressor = rl._compressor + rl._compressor = compressor + for chunk in chunks[0]: + rl.compress(chunk) + finally: + rl._compressor = oldcompressor + + benches = [ + (lambda: doread(), 'read'), + (lambda: doreadcachedfh(), 'read w/ reused fd'), + (lambda: doreadbatch(), 'read batch'), + (lambda: doreadbatchcachedfh(), 'read batch w/ reused fd'), + (lambda: dochunk(), 'chunk'), + (lambda: dochunkbatch(), 'chunk batch'), + ] + + for engine in sorted(engines): + compressor = util.compengines[engine].revlogcompressor() + benches.append((functools.partial(docompress, compressor), + 'compress w/ %s' % engine)) + + for fn, title in benches: + timer, fm = gettimer(ui, opts) + timer(fn, title=title) + fm.end() + @command('perfrevlogrevision', revlogopts + formatteropts + [('', 'cache', False, 'use caches instead of clearing')], '-c|-m|FILE REV') @@ -851,7 +1020,7 @@ chunkstart += (rev + 1) * iosize chunklength = length(rev) b = buffer(data, chunkstart - offset, chunklength) - revlog.decompress(b) + r.decompress(b) def dopatch(text, bins): if not cache: @@ -861,7 +1030,7 @@ def dohash(text): if not cache: r.clearcaches() - r._checkhash(text, node, rev) + r.checkhash(text, node, rev=rev) def dorevision(): if not cache: diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressiondict.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressiondict.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressiondict.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressiondict.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,247 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs) { + static char *kwlist[] = { "dict_size", "samples", "parameters", NULL }; + size_t capacity; + PyObject* samples; + Py_ssize_t samplesLen; + PyObject* parameters = NULL; + ZDICT_params_t zparams; + Py_ssize_t sampleIndex; + Py_ssize_t sampleSize; + PyObject* sampleItem; + size_t zresult; + void* sampleBuffer; + void* sampleOffset; + size_t samplesSize = 0; + size_t* sampleSizes; + void* dict; + ZstdCompressionDict* result; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "nO!|O!", kwlist, + &capacity, + &PyList_Type, &samples, + (PyObject*)&DictParametersType, ¶meters)) { + return NULL; + } + + /* Validate parameters first since it is easiest. */ + zparams.selectivityLevel = 0; + zparams.compressionLevel = 0; + zparams.notificationLevel = 0; + zparams.dictID = 0; + zparams.reserved[0] = 0; + zparams.reserved[1] = 0; + + if (parameters) { + /* TODO validate data ranges */ + zparams.selectivityLevel = PyLong_AsUnsignedLong(PyTuple_GetItem(parameters, 0)); + zparams.compressionLevel = PyLong_AsLong(PyTuple_GetItem(parameters, 1)); + zparams.notificationLevel = PyLong_AsUnsignedLong(PyTuple_GetItem(parameters, 2)); + zparams.dictID = PyLong_AsUnsignedLong(PyTuple_GetItem(parameters, 3)); + } + + /* Figure out the size of the raw samples */ + samplesLen = PyList_Size(samples); + for (sampleIndex = 0; sampleIndex < samplesLen; sampleIndex++) { + sampleItem = PyList_GetItem(samples, sampleIndex); + if (!PyBytes_Check(sampleItem)) { + PyErr_SetString(PyExc_ValueError, "samples must be bytes"); + /* TODO probably need to perform DECREF here */ + return NULL; + } + samplesSize += PyBytes_GET_SIZE(sampleItem); + } + + /* Now that we know the total size of the raw simples, we can allocate + a buffer for the raw data */ + sampleBuffer = PyMem_Malloc(samplesSize); + if (!sampleBuffer) { + PyErr_NoMemory(); + return NULL; + } + sampleSizes = PyMem_Malloc(samplesLen * sizeof(size_t)); + if (!sampleSizes) { + PyMem_Free(sampleBuffer); + PyErr_NoMemory(); + return NULL; + } + + sampleOffset = sampleBuffer; + /* Now iterate again and assemble the samples in the buffer */ + for (sampleIndex = 0; sampleIndex < samplesLen; sampleIndex++) { + sampleItem = PyList_GetItem(samples, sampleIndex); + sampleSize = PyBytes_GET_SIZE(sampleItem); + sampleSizes[sampleIndex] = sampleSize; + memcpy(sampleOffset, PyBytes_AS_STRING(sampleItem), sampleSize); + sampleOffset = (char*)sampleOffset + sampleSize; + } + + dict = PyMem_Malloc(capacity); + if (!dict) { + PyMem_Free(sampleSizes); + PyMem_Free(sampleBuffer); + PyErr_NoMemory(); + return NULL; + } + + zresult = ZDICT_trainFromBuffer_advanced(dict, capacity, + sampleBuffer, sampleSizes, (unsigned int)samplesLen, + zparams); + if (ZDICT_isError(zresult)) { + PyErr_Format(ZstdError, "Cannot train dict: %s", ZDICT_getErrorName(zresult)); + PyMem_Free(dict); + PyMem_Free(sampleSizes); + PyMem_Free(sampleBuffer); + return NULL; + } + + result = PyObject_New(ZstdCompressionDict, &ZstdCompressionDictType); + if (!result) { + return NULL; + } + + result->dictData = dict; + result->dictSize = zresult; + return result; +} + + +PyDoc_STRVAR(ZstdCompressionDict__doc__, +"ZstdCompressionDict(data) - Represents a computed compression dictionary\n" +"\n" +"This type holds the results of a computed Zstandard compression dictionary.\n" +"Instances are obtained by calling ``train_dictionary()`` or by passing bytes\n" +"obtained from another source into the constructor.\n" +); + +static int ZstdCompressionDict_init(ZstdCompressionDict* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + + self->dictData = NULL; + self->dictSize = 0; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return -1; + } + + self->dictData = PyMem_Malloc(sourceSize); + if (!self->dictData) { + PyErr_NoMemory(); + return -1; + } + + memcpy(self->dictData, source, sourceSize); + self->dictSize = sourceSize; + + return 0; + } + +static void ZstdCompressionDict_dealloc(ZstdCompressionDict* self) { + if (self->dictData) { + PyMem_Free(self->dictData); + self->dictData = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdCompressionDict_dict_id(ZstdCompressionDict* self) { + unsigned dictID = ZDICT_getDictID(self->dictData, self->dictSize); + + return PyLong_FromLong(dictID); +} + +static PyObject* ZstdCompressionDict_as_bytes(ZstdCompressionDict* self) { + return PyBytes_FromStringAndSize(self->dictData, self->dictSize); +} + +static PyMethodDef ZstdCompressionDict_methods[] = { + { "dict_id", (PyCFunction)ZstdCompressionDict_dict_id, METH_NOARGS, + PyDoc_STR("dict_id() -- obtain the numeric dictionary ID") }, + { "as_bytes", (PyCFunction)ZstdCompressionDict_as_bytes, METH_NOARGS, + PyDoc_STR("as_bytes() -- obtain the raw bytes constituting the dictionary data") }, + { NULL, NULL } +}; + +static Py_ssize_t ZstdCompressionDict_length(ZstdCompressionDict* self) { + return self->dictSize; +} + +static PySequenceMethods ZstdCompressionDict_sq = { + (lenfunc)ZstdCompressionDict_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + 0, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +PyTypeObject ZstdCompressionDictType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressionDict", /* tp_name */ + sizeof(ZstdCompressionDict), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressionDict_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &ZstdCompressionDict_sq, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressionDict__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressionDict_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ZstdCompressionDict_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressiondict_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressionDictType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressionDictType) < 0) { + return; + } + + Py_INCREF((PyObject*)&ZstdCompressionDictType); + PyModule_AddObject(mod, "ZstdCompressionDict", + (PyObject*)&ZstdCompressionDictType); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressionparams.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressionparams.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressionparams.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressionparams.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,226 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams) { + zparams->windowLog = params->windowLog; + zparams->chainLog = params->chainLog; + zparams->hashLog = params->hashLog; + zparams->searchLog = params->searchLog; + zparams->searchLength = params->searchLength; + zparams->targetLength = params->targetLength; + zparams->strategy = params->strategy; +} + +CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args) { + int compressionLevel; + unsigned PY_LONG_LONG sourceSize = 0; + Py_ssize_t dictSize = 0; + ZSTD_compressionParameters params; + CompressionParametersObject* result; + + if (!PyArg_ParseTuple(args, "i|Kn", &compressionLevel, &sourceSize, &dictSize)) { + return NULL; + } + + params = ZSTD_getCParams(compressionLevel, sourceSize, dictSize); + + result = PyObject_New(CompressionParametersObject, &CompressionParametersType); + if (!result) { + return NULL; + } + + result->windowLog = params.windowLog; + result->chainLog = params.chainLog; + result->hashLog = params.hashLog; + result->searchLog = params.searchLog; + result->searchLength = params.searchLength; + result->targetLength = params.targetLength; + result->strategy = params.strategy; + + return result; +} + +PyObject* estimate_compression_context_size(PyObject* self, PyObject* args) { + CompressionParametersObject* params; + ZSTD_compressionParameters zparams; + PyObject* result; + + if (!PyArg_ParseTuple(args, "O!", &CompressionParametersType, ¶ms)) { + return NULL; + } + + ztopy_compression_parameters(params, &zparams); + result = PyLong_FromSize_t(ZSTD_estimateCCtxSize(zparams)); + return result; +} + +PyDoc_STRVAR(CompressionParameters__doc__, +"CompressionParameters: low-level control over zstd compression"); + +static PyObject* CompressionParameters_new(PyTypeObject* subtype, PyObject* args, PyObject* kwargs) { + CompressionParametersObject* self; + unsigned windowLog; + unsigned chainLog; + unsigned hashLog; + unsigned searchLog; + unsigned searchLength; + unsigned targetLength; + unsigned strategy; + + if (!PyArg_ParseTuple(args, "IIIIIII", &windowLog, &chainLog, &hashLog, &searchLog, + &searchLength, &targetLength, &strategy)) { + return NULL; + } + + if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid window log value"); + return NULL; + } + + if (chainLog < ZSTD_CHAINLOG_MIN || chainLog > ZSTD_CHAINLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid chain log value"); + return NULL; + } + + if (hashLog < ZSTD_HASHLOG_MIN || hashLog > ZSTD_HASHLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid hash log value"); + return NULL; + } + + if (searchLog < ZSTD_SEARCHLOG_MIN || searchLog > ZSTD_SEARCHLOG_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid search log value"); + return NULL; + } + + if (searchLength < ZSTD_SEARCHLENGTH_MIN || searchLength > ZSTD_SEARCHLENGTH_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid search length value"); + return NULL; + } + + if (targetLength < ZSTD_TARGETLENGTH_MIN || targetLength > ZSTD_TARGETLENGTH_MAX) { + PyErr_SetString(PyExc_ValueError, "invalid target length value"); + return NULL; + } + + if (strategy < ZSTD_fast || strategy > ZSTD_btopt) { + PyErr_SetString(PyExc_ValueError, "invalid strategy value"); + return NULL; + } + + self = (CompressionParametersObject*)subtype->tp_alloc(subtype, 1); + if (!self) { + return NULL; + } + + self->windowLog = windowLog; + self->chainLog = chainLog; + self->hashLog = hashLog; + self->searchLog = searchLog; + self->searchLength = searchLength; + self->targetLength = targetLength; + self->strategy = strategy; + + return (PyObject*)self; +} + +static void CompressionParameters_dealloc(PyObject* self) { + PyObject_Del(self); +} + +static Py_ssize_t CompressionParameters_length(PyObject* self) { + return 7; +} + +static PyObject* CompressionParameters_item(PyObject* o, Py_ssize_t i) { + CompressionParametersObject* self = (CompressionParametersObject*)o; + + switch (i) { + case 0: + return PyLong_FromLong(self->windowLog); + case 1: + return PyLong_FromLong(self->chainLog); + case 2: + return PyLong_FromLong(self->hashLog); + case 3: + return PyLong_FromLong(self->searchLog); + case 4: + return PyLong_FromLong(self->searchLength); + case 5: + return PyLong_FromLong(self->targetLength); + case 6: + return PyLong_FromLong(self->strategy); + default: + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +static PySequenceMethods CompressionParameters_sq = { + CompressionParameters_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + CompressionParameters_item, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +PyTypeObject CompressionParametersType = { + PyVarObject_HEAD_INIT(NULL, 0) + "CompressionParameters", /* tp_name */ + sizeof(CompressionParametersObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)CompressionParameters_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &CompressionParameters_sq, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + CompressionParameters__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + CompressionParameters_new, /* tp_new */ +}; + +void compressionparams_module_init(PyObject* mod) { + Py_TYPE(&CompressionParametersType) = &PyType_Type; + if (PyType_Ready(&CompressionParametersType) < 0) { + return; + } + + Py_IncRef((PyObject*)&CompressionParametersType); + PyModule_AddObject(mod, "CompressionParameters", + (PyObject*)&CompressionParametersType); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressionwriter.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressionwriter.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressionwriter.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressionwriter.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,288 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdCompresssionWriter__doc__, +"""A context manager used for writing compressed output to a writer.\n" +); + +static void ZstdCompressionWriter_dealloc(ZstdCompressionWriter* self) { + Py_XDECREF(self->compressor); + Py_XDECREF(self->writer); + + if (self->cstream) { + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdCompressionWriter_enter(ZstdCompressionWriter* self) { + if (self->entered) { + PyErr_SetString(ZstdError, "cannot __enter__ multiple times"); + return NULL; + } + + self->cstream = CStream_from_ZstdCompressor(self->compressor, self->sourceSize); + if (!self->cstream) { + return NULL; + } + + self->entered = 1; + + Py_INCREF(self); + return (PyObject*)self; +} + +static PyObject* ZstdCompressionWriter_exit(ZstdCompressionWriter* self, PyObject* args) { + PyObject* exc_type; + PyObject* exc_value; + PyObject* exc_tb; + size_t zresult; + + ZSTD_outBuffer output; + PyObject* res; + + if (!PyArg_ParseTuple(args, "OOO", &exc_type, &exc_value, &exc_tb)) { + return NULL; + } + + self->entered = 0; + + if (self->cstream && exc_type == Py_None && exc_value == Py_None && + exc_tb == Py_None) { + + output.dst = PyMem_Malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + while (1) { + zresult = ZSTD_endStream(self->cstream, &output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + PyMem_Free(output.dst); + return NULL; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + } + + if (!zresult) { + break; + } + + output.pos = 0; + } + + PyMem_Free(output.dst); + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + Py_RETURN_FALSE; +} + +static PyObject* ZstdCompressionWriter_memory_size(ZstdCompressionWriter* self) { + if (!self->cstream) { + PyErr_SetString(ZstdError, "cannot determine size of an inactive compressor; " + "call when a context manager is active"); + return NULL; + } + + return PyLong_FromSize_t(ZSTD_sizeof_CStream(self->cstream)); +} + +static PyObject* ZstdCompressionWriter_write(ZstdCompressionWriter* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t zresult; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + PyObject* res; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + if (!self->entered) { + PyErr_SetString(ZstdError, "compress must be called from an active context manager"); + return NULL; + } + + output.dst = PyMem_Malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + while ((ssize_t)input.pos < sourceSize) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyMem_Free(output.dst); + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + /* Copy data from output buffer to writer. */ + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + } + output.pos = 0; + } + + PyMem_Free(output.dst); + + /* TODO return bytes written */ + Py_RETURN_NONE; +} + +static PyObject* ZstdCompressionWriter_flush(ZstdCompressionWriter* self, PyObject* args) { + size_t zresult; + ZSTD_outBuffer output; + PyObject* res; + + if (!self->entered) { + PyErr_SetString(ZstdError, "flush must be called from an active context manager"); + return NULL; + } + + output.dst = PyMem_Malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + while (1) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_flushStream(self->cstream, &output); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyMem_Free(output.dst); + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + if (!output.pos) { + break; + } + + /* Copy data from output buffer to writer. */ + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + } + output.pos = 0; + } + + PyMem_Free(output.dst); + + /* TODO return bytes written */ + Py_RETURN_NONE; +} + +static PyMethodDef ZstdCompressionWriter_methods[] = { + { "__enter__", (PyCFunction)ZstdCompressionWriter_enter, METH_NOARGS, + PyDoc_STR("Enter a compression context.") }, + { "__exit__", (PyCFunction)ZstdCompressionWriter_exit, METH_VARARGS, + PyDoc_STR("Exit a compression context.") }, + { "memory_size", (PyCFunction)ZstdCompressionWriter_memory_size, METH_NOARGS, + PyDoc_STR("Obtain the memory size of the underlying compressor") }, + { "write", (PyCFunction)ZstdCompressionWriter_write, METH_VARARGS, + PyDoc_STR("Compress data") }, + { "flush", (PyCFunction)ZstdCompressionWriter_flush, METH_NOARGS, + PyDoc_STR("Flush data and finish a zstd frame") }, + { NULL, NULL } +}; + +PyTypeObject ZstdCompressionWriterType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressionWriter", /* tp_name */ + sizeof(ZstdCompressionWriter), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressionWriter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompresssionWriter__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressionWriter_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressionwriter_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressionWriterType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressionWriterType) < 0) { + return; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressobj.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressobj.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressobj.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressobj.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,250 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdCompressionObj__doc__, +"Perform compression using a standard library compatible API.\n" +); + +static void ZstdCompressionObj_dealloc(ZstdCompressionObj* self) { + PyMem_Free(self->output.dst); + self->output.dst = NULL; + + if (self->cstream) { + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + Py_XDECREF(self->compressor); + + PyObject_Del(self); +} + +static PyObject* ZstdCompressionObj_compress(ZstdCompressionObj* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + ZSTD_inBuffer input; + size_t zresult; + PyObject* result = NULL; + Py_ssize_t resultSize = 0; + + if (self->finished) { + PyErr_SetString(ZstdError, "cannot call compress() after compressor finished"); + return NULL; + } + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + while ((ssize_t)input.pos < sourceSize) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &self->output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + if (self->output.pos) { + if (result) { + resultSize = PyBytes_GET_SIZE(result); + if (-1 == _PyBytes_Resize(&result, resultSize + self->output.pos)) { + return NULL; + } + + memcpy(PyBytes_AS_STRING(result) + resultSize, + self->output.dst, self->output.pos); + } + else { + result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + if (!result) { + return NULL; + } + } + + self->output.pos = 0; + } + } + + if (result) { + return result; + } + else { + return PyBytes_FromString(""); + } +} + +static PyObject* ZstdCompressionObj_flush(ZstdCompressionObj* self, PyObject* args) { + int flushMode = compressorobj_flush_finish; + size_t zresult; + PyObject* result = NULL; + Py_ssize_t resultSize = 0; + + if (!PyArg_ParseTuple(args, "|i", &flushMode)) { + return NULL; + } + + if (flushMode != compressorobj_flush_finish && flushMode != compressorobj_flush_block) { + PyErr_SetString(PyExc_ValueError, "flush mode not recognized"); + return NULL; + } + + if (self->finished) { + PyErr_SetString(ZstdError, "compressor object already finished"); + return NULL; + } + + assert(self->output.pos == 0); + + if (flushMode == compressorobj_flush_block) { + /* The output buffer is of size ZSTD_CStreamOutSize(), which is + guaranteed to hold a full block. */ + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_flushStream(self->cstream, &self->output); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + /* Output buffer is guaranteed to hold full block. */ + assert(zresult == 0); + + if (self->output.pos) { + result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + if (!result) { + return NULL; + } + } + + self->output.pos = 0; + + if (result) { + return result; + } + else { + return PyBytes_FromString(""); + } + } + + assert(flushMode == compressorobj_flush_finish); + self->finished = 1; + + while (1) { + zresult = ZSTD_endStream(self->cstream, &self->output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + if (self->output.pos) { + if (result) { + resultSize = PyBytes_GET_SIZE(result); + if (-1 == _PyBytes_Resize(&result, resultSize + self->output.pos)) { + return NULL; + } + + memcpy(PyBytes_AS_STRING(result) + resultSize, + self->output.dst, self->output.pos); + } + else { + result = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + if (!result) { + return NULL; + } + } + + self->output.pos = 0; + } + + if (!zresult) { + break; + } + } + + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + + if (result) { + return result; + } + else { + return PyBytes_FromString(""); + } +} + +static PyMethodDef ZstdCompressionObj_methods[] = { + { "compress", (PyCFunction)ZstdCompressionObj_compress, METH_VARARGS, + PyDoc_STR("compress data") }, + { "flush", (PyCFunction)ZstdCompressionObj_flush, METH_VARARGS, + PyDoc_STR("finish compression operation") }, + { NULL, NULL } +}; + +PyTypeObject ZstdCompressionObjType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressionObj", /* tp_name */ + sizeof(ZstdCompressionObj), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressionObj_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressionObj__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressionObj_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressobj_module_init(PyObject* module) { + Py_TYPE(&ZstdCompressionObjType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressionObjType) < 0) { + return; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressor.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressor.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressor.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressor.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,791 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +int populate_cdict(ZstdCompressor* compressor, void* dictData, size_t dictSize, ZSTD_parameters* zparams) { + ZSTD_customMem zmem; + assert(!compressor->cdict); + Py_BEGIN_ALLOW_THREADS + memset(&zmem, 0, sizeof(zmem)); + compressor->cdict = ZSTD_createCDict_advanced(compressor->dict->dictData, + compressor->dict->dictSize, *zparams, zmem); + Py_END_ALLOW_THREADS + + if (!compressor->cdict) { + PyErr_SetString(ZstdError, "could not create compression dictionary"); + return 1; + } + + return 0; +} + +/** +* Initialize a zstd CStream from a ZstdCompressor instance. +* +* Returns a ZSTD_CStream on success or NULL on failure. If NULL, a Python +* exception will be set. +*/ +ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize) { + ZSTD_CStream* cstream; + ZSTD_parameters zparams; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + + cstream = ZSTD_createCStream(); + if (!cstream) { + PyErr_SetString(ZstdError, "cannot create CStream"); + return NULL; + } + + if (compressor->dict) { + dictData = compressor->dict->dictData; + dictSize = compressor->dict->dictSize; + } + + memset(&zparams, 0, sizeof(zparams)); + if (compressor->cparams) { + ztopy_compression_parameters(compressor->cparams, &zparams.cParams); + /* Do NOT call ZSTD_adjustCParams() here because the compression params + come from the user. */ + } + else { + zparams.cParams = ZSTD_getCParams(compressor->compressionLevel, sourceSize, dictSize); + } + + zparams.fParams = compressor->fparams; + + zresult = ZSTD_initCStream_advanced(cstream, dictData, dictSize, zparams, sourceSize); + + if (ZSTD_isError(zresult)) { + ZSTD_freeCStream(cstream); + PyErr_Format(ZstdError, "cannot init CStream: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + return cstream; +} + +PyDoc_STRVAR(ZstdCompressor__doc__, +"ZstdCompressor(level=None, dict_data=None, compression_params=None)\n" +"\n" +"Create an object used to perform Zstandard compression.\n" +"\n" +"An instance can compress data various ways. Instances can be used multiple\n" +"times. Each compression operation will use the compression parameters\n" +"defined at construction time.\n" +"\n" +"Compression can be configured via the following names arguments:\n" +"\n" +"level\n" +" Integer compression level.\n" +"dict_data\n" +" A ``ZstdCompressionDict`` to be used to compress with dictionary data.\n" +"compression_params\n" +" A ``CompressionParameters`` instance defining low-level compression" +" parameters. If defined, this will overwrite the ``level`` argument.\n" +"write_checksum\n" +" If True, a 4 byte content checksum will be written with the compressed\n" +" data, allowing the decompressor to perform content verification.\n" +"write_content_size\n" +" If True, the decompressed content size will be included in the header of\n" +" the compressed data. This data will only be written if the compressor\n" +" knows the size of the input data.\n" +"write_dict_id\n" +" Determines whether the dictionary ID will be written into the compressed\n" +" data. Defaults to True. Only adds content to the compressed data if\n" +" a dictionary is being used.\n" +); + +static int ZstdCompressor_init(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "level", + "dict_data", + "compression_params", + "write_checksum", + "write_content_size", + "write_dict_id", + NULL + }; + + int level = 3; + ZstdCompressionDict* dict = NULL; + CompressionParametersObject* params = NULL; + PyObject* writeChecksum = NULL; + PyObject* writeContentSize = NULL; + PyObject* writeDictID = NULL; + + self->cctx = NULL; + self->dict = NULL; + self->cparams = NULL; + self->cdict = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|iO!O!OOO", kwlist, + &level, &ZstdCompressionDictType, &dict, + &CompressionParametersType, ¶ms, + &writeChecksum, &writeContentSize, &writeDictID)) { + return -1; + } + + if (level < 1) { + PyErr_SetString(PyExc_ValueError, "level must be greater than 0"); + return -1; + } + + if (level > ZSTD_maxCLevel()) { + PyErr_Format(PyExc_ValueError, "level must be less than %d", + ZSTD_maxCLevel() + 1); + return -1; + } + + /* We create a ZSTD_CCtx for reuse among multiple operations to reduce the + overhead of each compression operation. */ + self->cctx = ZSTD_createCCtx(); + if (!self->cctx) { + PyErr_NoMemory(); + return -1; + } + + self->compressionLevel = level; + + if (dict) { + self->dict = dict; + Py_INCREF(dict); + } + + if (params) { + self->cparams = params; + Py_INCREF(params); + } + + memset(&self->fparams, 0, sizeof(self->fparams)); + + if (writeChecksum && PyObject_IsTrue(writeChecksum)) { + self->fparams.checksumFlag = 1; + } + if (writeContentSize && PyObject_IsTrue(writeContentSize)) { + self->fparams.contentSizeFlag = 1; + } + if (writeDictID && PyObject_Not(writeDictID)) { + self->fparams.noDictIDFlag = 1; + } + + return 0; +} + +static void ZstdCompressor_dealloc(ZstdCompressor* self) { + Py_XDECREF(self->cparams); + Py_XDECREF(self->dict); + + if (self->cdict) { + ZSTD_freeCDict(self->cdict); + self->cdict = NULL; + } + + if (self->cctx) { + ZSTD_freeCCtx(self->cctx); + self->cctx = NULL; + } + + PyObject_Del(self); +} + +PyDoc_STRVAR(ZstdCompressor_copy_stream__doc__, +"copy_stream(ifh, ofh[, size=0, read_size=default, write_size=default])\n" +"compress data between streams\n" +"\n" +"Data will be read from ``ifh``, compressed, and written to ``ofh``.\n" +"``ifh`` must have a ``read(size)`` method. ``ofh`` must have a ``write(data)``\n" +"method.\n" +"\n" +"An optional ``size`` argument specifies the size of the source stream.\n" +"If defined, compression parameters will be tuned based on the size.\n" +"\n" +"Optional arguments ``read_size`` and ``write_size`` define the chunk sizes\n" +"of ``read()`` and ``write()`` operations, respectively. By default, they use\n" +"the default compression stream input and output sizes, respectively.\n" +); + +static PyObject* ZstdCompressor_copy_stream(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "ifh", + "ofh", + "size", + "read_size", + "write_size", + NULL + }; + + PyObject* source; + PyObject* dest; + Py_ssize_t sourceSize = 0; + size_t inSize = ZSTD_CStreamInSize(); + size_t outSize = ZSTD_CStreamOutSize(); + ZSTD_CStream* cstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + Py_ssize_t totalRead = 0; + Py_ssize_t totalWrite = 0; + char* readBuffer; + Py_ssize_t readSize; + PyObject* readResult; + PyObject* res = NULL; + size_t zresult; + PyObject* writeResult; + PyObject* totalReadPy; + PyObject* totalWritePy; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|nkk", kwlist, &source, &dest, &sourceSize, + &inSize, &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(source, "read")) { + PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); + return NULL; + } + + if (!PyObject_HasAttrString(dest, "write")) { + PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); + return NULL; + } + + /* Prevent free on uninitialized memory in finally. */ + output.dst = NULL; + + cstream = CStream_from_ZstdCompressor(self, sourceSize); + if (!cstream) { + res = NULL; + goto finally; + } + + output.dst = PyMem_Malloc(outSize); + if (!output.dst) { + PyErr_NoMemory(); + res = NULL; + goto finally; + } + output.size = outSize; + output.pos = 0; + + while (1) { + /* Try to read from source stream. */ + readResult = PyObject_CallMethod(source, "read", "n", inSize); + if (!readResult) { + PyErr_SetString(ZstdError, "could not read() from source"); + goto finally; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + + /* If no data was read, we're at EOF. */ + if (0 == readSize) { + break; + } + + totalRead += readSize; + + /* Send data to compressor */ + input.src = readBuffer; + input.size = readSize; + input.pos = 0; + + while (input.pos < input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(cstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + res = NULL; + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + goto finally; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + writeResult = PyObject_CallMethod(dest, "write", "y#", +#else + writeResult = PyObject_CallMethod(dest, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(writeResult); + totalWrite += output.pos; + output.pos = 0; + } + } + } + + /* We've finished reading. Now flush the compressor stream. */ + while (1) { + zresult = ZSTD_endStream(cstream, &output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + res = NULL; + goto finally; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + writeResult = PyObject_CallMethod(dest, "write", "y#", +#else + writeResult = PyObject_CallMethod(dest, "write", "s#", +#endif + output.dst, output.pos); + totalWrite += output.pos; + Py_XDECREF(writeResult); + output.pos = 0; + } + + if (!zresult) { + break; + } + } + + ZSTD_freeCStream(cstream); + cstream = NULL; + + totalReadPy = PyLong_FromSsize_t(totalRead); + totalWritePy = PyLong_FromSsize_t(totalWrite); + res = PyTuple_Pack(2, totalReadPy, totalWritePy); + Py_DecRef(totalReadPy); + Py_DecRef(totalWritePy); + +finally: + if (output.dst) { + PyMem_Free(output.dst); + } + + if (cstream) { + ZSTD_freeCStream(cstream); + } + + return res; +} + +PyDoc_STRVAR(ZstdCompressor_compress__doc__, +"compress(data, allow_empty=False)\n" +"\n" +"Compress data in a single operation.\n" +"\n" +"This is the simplest mechanism to perform compression: simply pass in a\n" +"value and get a compressed value back. It is almost the most prone to abuse.\n" +"The input and output values must fit in memory, so passing in very large\n" +"values can result in excessive memory usage. For this reason, one of the\n" +"streaming based APIs is preferred for larger values.\n" +); + +static PyObject* ZstdCompressor_compress(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "data", + "allow_empty", + NULL + }; + + const char* source; + Py_ssize_t sourceSize; + PyObject* allowEmpty = NULL; + size_t destSize; + PyObject* output; + char* dest; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + ZSTD_parameters zparams; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|O", +#else + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|O", +#endif + kwlist, &source, &sourceSize, &allowEmpty)) { + return NULL; + } + + /* Limitation in zstd C API doesn't let decompression side distinguish + between content size of 0 and unknown content size. This can make round + tripping via Python difficult. Until this is fixed, require a flag + to fire the footgun. + https://github.com/indygreg/python-zstandard/issues/11 */ + if (0 == sourceSize && self->fparams.contentSizeFlag + && (!allowEmpty || PyObject_Not(allowEmpty))) { + PyErr_SetString(PyExc_ValueError, "cannot write empty inputs when writing content sizes"); + return NULL; + } + + destSize = ZSTD_compressBound(sourceSize); + output = PyBytes_FromStringAndSize(NULL, destSize); + if (!output) { + return NULL; + } + + dest = PyBytes_AsString(output); + + if (self->dict) { + dictData = self->dict->dictData; + dictSize = self->dict->dictSize; + } + + memset(&zparams, 0, sizeof(zparams)); + if (!self->cparams) { + zparams.cParams = ZSTD_getCParams(self->compressionLevel, sourceSize, dictSize); + } + else { + ztopy_compression_parameters(self->cparams, &zparams.cParams); + /* Do NOT call ZSTD_adjustCParams() here because the compression params + come from the user. */ + } + + zparams.fParams = self->fparams; + + /* The raw dict data has to be processed before it can be used. Since this + adds overhead - especially if multiple dictionary compression operations + are performed on the same ZstdCompressor instance - we create a + ZSTD_CDict once and reuse it for all operations. + + Note: the compression parameters used for the first invocation (possibly + derived from the source size) will be reused on all subsequent invocations. + https://github.com/facebook/zstd/issues/358 contains more info. We could + potentially add an argument somewhere to control this behavior. + */ + if (dictData && !self->cdict) { + if (populate_cdict(self, dictData, dictSize, &zparams)) { + Py_DECREF(output); + return NULL; + } + } + + Py_BEGIN_ALLOW_THREADS + /* By avoiding ZSTD_compress(), we don't necessarily write out content + size. This means the argument to ZstdCompressor to control frame + parameters is honored. */ + if (self->cdict) { + zresult = ZSTD_compress_usingCDict(self->cctx, dest, destSize, + source, sourceSize, self->cdict); + } + else { + zresult = ZSTD_compress_advanced(self->cctx, dest, destSize, + source, sourceSize, dictData, dictSize, zparams); + } + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "cannot compress: %s", ZSTD_getErrorName(zresult)); + Py_CLEAR(output); + return NULL; + } + else { + Py_SIZE(output) = zresult; + } + + return output; +} + +PyDoc_STRVAR(ZstdCompressionObj__doc__, +"compressobj()\n" +"\n" +"Return an object exposing ``compress(data)`` and ``flush()`` methods.\n" +"\n" +"The returned object exposes an API similar to ``zlib.compressobj`` and\n" +"``bz2.BZ2Compressor`` so that callers can swap in the zstd compressor\n" +"without changing how compression is performed.\n" +); + +static ZstdCompressionObj* ZstdCompressor_compressobj(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "size", + NULL + }; + + Py_ssize_t inSize = 0; + size_t outSize = ZSTD_CStreamOutSize(); + ZstdCompressionObj* result = PyObject_New(ZstdCompressionObj, &ZstdCompressionObjType); + if (!result) { + return NULL; + } + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|n", kwlist, &inSize)) { + return NULL; + } + + result->cstream = CStream_from_ZstdCompressor(self, inSize); + if (!result->cstream) { + Py_DECREF(result); + return NULL; + } + + result->output.dst = PyMem_Malloc(outSize); + if (!result->output.dst) { + PyErr_NoMemory(); + Py_DECREF(result); + return NULL; + } + result->output.size = outSize; + result->output.pos = 0; + + result->compressor = self; + Py_INCREF(result->compressor); + + result->finished = 0; + + return result; +} + +PyDoc_STRVAR(ZstdCompressor_read_from__doc__, +"read_from(reader, [size=0, read_size=default, write_size=default])\n" +"Read uncompress data from a reader and return an iterator\n" +"\n" +"Returns an iterator of compressed data produced from reading from ``reader``.\n" +"\n" +"Uncompressed data will be obtained from ``reader`` by calling the\n" +"``read(size)`` method of it. The source data will be streamed into a\n" +"compressor. As compressed data is available, it will be exposed to the\n" +"iterator.\n" +"\n" +"Data is read from the source in chunks of ``read_size``. Compressed chunks\n" +"are at most ``write_size`` bytes. Both values default to the zstd input and\n" +"and output defaults, respectively.\n" +"\n" +"The caller is partially in control of how fast data is fed into the\n" +"compressor by how it consumes the returned iterator. The compressor will\n" +"not consume from the reader unless the caller consumes from the iterator.\n" +); + +static ZstdCompressorIterator* ZstdCompressor_read_from(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "reader", + "size", + "read_size", + "write_size", + NULL + }; + + PyObject* reader; + Py_ssize_t sourceSize = 0; + size_t inSize = ZSTD_CStreamInSize(); + size_t outSize = ZSTD_CStreamOutSize(); + ZstdCompressorIterator* result; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nkk", kwlist, &reader, &sourceSize, + &inSize, &outSize)) { + return NULL; + } + + result = PyObject_New(ZstdCompressorIterator, &ZstdCompressorIteratorType); + if (!result) { + return NULL; + } + + result->compressor = NULL; + result->reader = NULL; + result->buffer = NULL; + result->cstream = NULL; + result->input.src = NULL; + result->output.dst = NULL; + result->readResult = NULL; + + if (PyObject_HasAttrString(reader, "read")) { + result->reader = reader; + Py_INCREF(result->reader); + } + else if (1 == PyObject_CheckBuffer(reader)) { + result->buffer = PyMem_Malloc(sizeof(Py_buffer)); + if (!result->buffer) { + goto except; + } + + memset(result->buffer, 0, sizeof(Py_buffer)); + + if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { + goto except; + } + + result->bufferOffset = 0; + sourceSize = result->buffer->len; + } + else { + PyErr_SetString(PyExc_ValueError, + "must pass an object with a read() method or conforms to buffer protocol"); + goto except; + } + + result->compressor = self; + Py_INCREF(result->compressor); + + result->sourceSize = sourceSize; + result->cstream = CStream_from_ZstdCompressor(self, sourceSize); + if (!result->cstream) { + goto except; + } + + result->inSize = inSize; + result->outSize = outSize; + + result->output.dst = PyMem_Malloc(outSize); + if (!result->output.dst) { + PyErr_NoMemory(); + goto except; + } + result->output.size = outSize; + result->output.pos = 0; + + result->input.src = NULL; + result->input.size = 0; + result->input.pos = 0; + + result->finishedInput = 0; + result->finishedOutput = 0; + + goto finally; + +except: + if (result->cstream) { + ZSTD_freeCStream(result->cstream); + result->cstream = NULL; + } + + Py_DecRef((PyObject*)result->compressor); + Py_DecRef(result->reader); + + Py_DECREF(result); + result = NULL; + +finally: + return result; +} + +PyDoc_STRVAR(ZstdCompressor_write_to___doc__, +"Create a context manager to write compressed data to an object.\n" +"\n" +"The passed object must have a ``write()`` method.\n" +"\n" +"The caller feeds input data to the object by calling ``compress(data)``.\n" +"Compressed data is written to the argument given to this function.\n" +"\n" +"The function takes an optional ``size`` argument indicating the total size\n" +"of the eventual input. If specified, the size will influence compression\n" +"parameter tuning and could result in the size being written into the\n" +"header of the compressed data.\n" +"\n" +"An optional ``write_size`` argument is also accepted. It defines the maximum\n" +"byte size of chunks fed to ``write()``. By default, it uses the zstd default\n" +"for a compressor output stream.\n" +); + +static ZstdCompressionWriter* ZstdCompressor_write_to(ZstdCompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "writer", + "size", + "write_size", + NULL + }; + + PyObject* writer; + ZstdCompressionWriter* result; + Py_ssize_t sourceSize = 0; + size_t outSize = ZSTD_CStreamOutSize(); + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|nk", kwlist, &writer, &sourceSize, + &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(writer, "write")) { + PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); + return NULL; + } + + result = PyObject_New(ZstdCompressionWriter, &ZstdCompressionWriterType); + if (!result) { + return NULL; + } + + result->compressor = self; + Py_INCREF(result->compressor); + + result->writer = writer; + Py_INCREF(result->writer); + + result->sourceSize = sourceSize; + + result->outSize = outSize; + + result->entered = 0; + result->cstream = NULL; + + return result; +} + +static PyMethodDef ZstdCompressor_methods[] = { + { "compress", (PyCFunction)ZstdCompressor_compress, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_compress__doc__ }, + { "compressobj", (PyCFunction)ZstdCompressor_compressobj, + METH_VARARGS | METH_KEYWORDS, ZstdCompressionObj__doc__ }, + { "copy_stream", (PyCFunction)ZstdCompressor_copy_stream, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_copy_stream__doc__ }, + { "read_from", (PyCFunction)ZstdCompressor_read_from, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_read_from__doc__ }, + { "write_to", (PyCFunction)ZstdCompressor_write_to, + METH_VARARGS | METH_KEYWORDS, ZstdCompressor_write_to___doc__ }, + { NULL, NULL } +}; + +PyTypeObject ZstdCompressorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressor", /* tp_name */ + sizeof(ZstdCompressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressor_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdCompressor_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)ZstdCompressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressor_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressorType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressorType) < 0) { + return; + } + + Py_INCREF((PyObject*)&ZstdCompressorType); + PyModule_AddObject(mod, "ZstdCompressor", + (PyObject*)&ZstdCompressorType); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressoriterator.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressoriterator.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/compressoriterator.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/compressoriterator.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,234 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdCompressorIterator__doc__, +"Represents an iterator of compressed data.\n" +); + +static void ZstdCompressorIterator_dealloc(ZstdCompressorIterator* self) { + Py_XDECREF(self->readResult); + Py_XDECREF(self->compressor); + Py_XDECREF(self->reader); + + if (self->buffer) { + PyBuffer_Release(self->buffer); + PyMem_FREE(self->buffer); + self->buffer = NULL; + } + + if (self->cstream) { + ZSTD_freeCStream(self->cstream); + self->cstream = NULL; + } + + if (self->output.dst) { + PyMem_Free(self->output.dst); + self->output.dst = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdCompressorIterator_iter(PyObject* self) { + Py_INCREF(self); + return self; +} + +static PyObject* ZstdCompressorIterator_iternext(ZstdCompressorIterator* self) { + size_t zresult; + PyObject* readResult = NULL; + PyObject* chunk; + char* readBuffer; + Py_ssize_t readSize = 0; + Py_ssize_t bufferRemaining; + + if (self->finishedOutput) { + PyErr_SetString(PyExc_StopIteration, "output flushed"); + return NULL; + } + +feedcompressor: + + /* If we have data left in the input, consume it. */ + if (self->input.pos < self->input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &self->output, &self->input); + Py_END_ALLOW_THREADS + + /* Release the Python object holding the input buffer. */ + if (self->input.pos == self->input.size) { + self->input.src = NULL; + self->input.pos = 0; + self->input.size = 0; + Py_DECREF(self->readResult); + self->readResult = NULL; + } + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + /* If it produced output data, emit it. */ + if (self->output.pos) { + chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + self->output.pos = 0; + return chunk; + } + } + + /* We should never have output data sitting around after a previous call. */ + assert(self->output.pos == 0); + + /* The code above should have either emitted a chunk and returned or consumed + the entire input buffer. So the state of the input buffer is not + relevant. */ + if (!self->finishedInput) { + if (self->reader) { + readResult = PyObject_CallMethod(self->reader, "read", "I", self->inSize); + if (!readResult) { + PyErr_SetString(ZstdError, "could not read() from source"); + return NULL; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + } + else { + assert(self->buffer && self->buffer->buf); + + /* Only support contiguous C arrays. */ + assert(self->buffer->strides == NULL && self->buffer->suboffsets == NULL); + assert(self->buffer->itemsize == 1); + + readBuffer = (char*)self->buffer->buf + self->bufferOffset; + bufferRemaining = self->buffer->len - self->bufferOffset; + readSize = min(bufferRemaining, (Py_ssize_t)self->inSize); + self->bufferOffset += readSize; + } + + if (0 == readSize) { + Py_XDECREF(readResult); + self->finishedInput = 1; + } + else { + self->readResult = readResult; + } + } + + /* EOF */ + if (0 == readSize) { + zresult = ZSTD_endStream(self->cstream, &self->output); + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "error ending compression stream: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + assert(self->output.pos); + + if (0 == zresult) { + self->finishedOutput = 1; + } + + chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + self->output.pos = 0; + return chunk; + } + + /* New data from reader. Feed into compressor. */ + self->input.src = readBuffer; + self->input.size = readSize; + self->input.pos = 0; + + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_compressStream(self->cstream, &self->output, &self->input); + Py_END_ALLOW_THREADS + + /* The input buffer currently points to memory managed by Python + (readBuffer). This object was allocated by this function. If it wasn't + fully consumed, we need to release it in a subsequent function call. + If it is fully consumed, do that now. + */ + if (self->input.pos == self->input.size) { + self->input.src = NULL; + self->input.pos = 0; + self->input.size = 0; + Py_XDECREF(self->readResult); + self->readResult = NULL; + } + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd compress error: %s", ZSTD_getErrorName(zresult)); + return NULL; + } + + assert(self->input.pos <= self->input.size); + + /* If we didn't write anything, start the process over. */ + if (0 == self->output.pos) { + goto feedcompressor; + } + + chunk = PyBytes_FromStringAndSize(self->output.dst, self->output.pos); + self->output.pos = 0; + return chunk; +} + +PyTypeObject ZstdCompressorIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdCompressorIterator", /* tp_name */ + sizeof(ZstdCompressorIterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdCompressorIterator_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdCompressorIterator__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + ZstdCompressorIterator_iter, /* tp_iter */ + (iternextfunc)ZstdCompressorIterator_iternext, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void compressoriterator_module_init(PyObject* mod) { + Py_TYPE(&ZstdCompressorIteratorType) = &PyType_Type; + if (PyType_Ready(&ZstdCompressorIteratorType) < 0) { + return; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/constants.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/constants.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/constants.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/constants.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,87 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +static char frame_header[] = { + '\x28', + '\xb5', + '\x2f', + '\xfd', +}; + +void constants_module_init(PyObject* mod) { + PyObject* version; + PyObject* zstdVersion; + PyObject* frameHeader; + +#if PY_MAJOR_VERSION >= 3 + version = PyUnicode_FromString(PYTHON_ZSTANDARD_VERSION); +#else + version = PyString_FromString(PYTHON_ZSTANDARD_VERSION); +#endif + Py_INCREF(version); + PyModule_AddObject(mod, "__version__", version); + + ZstdError = PyErr_NewException("zstd.ZstdError", NULL, NULL); + PyModule_AddObject(mod, "ZstdError", ZstdError); + + PyModule_AddIntConstant(mod, "COMPRESSOBJ_FLUSH_FINISH", compressorobj_flush_finish); + PyModule_AddIntConstant(mod, "COMPRESSOBJ_FLUSH_BLOCK", compressorobj_flush_block); + + /* For now, the version is a simple tuple instead of a dedicated type. */ + zstdVersion = PyTuple_New(3); + PyTuple_SetItem(zstdVersion, 0, PyLong_FromLong(ZSTD_VERSION_MAJOR)); + PyTuple_SetItem(zstdVersion, 1, PyLong_FromLong(ZSTD_VERSION_MINOR)); + PyTuple_SetItem(zstdVersion, 2, PyLong_FromLong(ZSTD_VERSION_RELEASE)); + Py_IncRef(zstdVersion); + PyModule_AddObject(mod, "ZSTD_VERSION", zstdVersion); + + frameHeader = PyBytes_FromStringAndSize(frame_header, sizeof(frame_header)); + if (frameHeader) { + PyModule_AddObject(mod, "FRAME_HEADER", frameHeader); + } + else { + PyErr_Format(PyExc_ValueError, "could not create frame header object"); + } + + PyModule_AddIntConstant(mod, "MAX_COMPRESSION_LEVEL", ZSTD_maxCLevel()); + PyModule_AddIntConstant(mod, "COMPRESSION_RECOMMENDED_INPUT_SIZE", + (long)ZSTD_CStreamInSize()); + PyModule_AddIntConstant(mod, "COMPRESSION_RECOMMENDED_OUTPUT_SIZE", + (long)ZSTD_CStreamOutSize()); + PyModule_AddIntConstant(mod, "DECOMPRESSION_RECOMMENDED_INPUT_SIZE", + (long)ZSTD_DStreamInSize()); + PyModule_AddIntConstant(mod, "DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE", + (long)ZSTD_DStreamOutSize()); + + PyModule_AddIntConstant(mod, "MAGIC_NUMBER", ZSTD_MAGICNUMBER); + PyModule_AddIntConstant(mod, "WINDOWLOG_MIN", ZSTD_WINDOWLOG_MIN); + PyModule_AddIntConstant(mod, "WINDOWLOG_MAX", ZSTD_WINDOWLOG_MAX); + PyModule_AddIntConstant(mod, "CHAINLOG_MIN", ZSTD_CHAINLOG_MIN); + PyModule_AddIntConstant(mod, "CHAINLOG_MAX", ZSTD_CHAINLOG_MAX); + PyModule_AddIntConstant(mod, "HASHLOG_MIN", ZSTD_HASHLOG_MIN); + PyModule_AddIntConstant(mod, "HASHLOG_MAX", ZSTD_HASHLOG_MAX); + PyModule_AddIntConstant(mod, "HASHLOG3_MAX", ZSTD_HASHLOG3_MAX); + PyModule_AddIntConstant(mod, "SEARCHLOG_MIN", ZSTD_SEARCHLOG_MIN); + PyModule_AddIntConstant(mod, "SEARCHLOG_MAX", ZSTD_SEARCHLOG_MAX); + PyModule_AddIntConstant(mod, "SEARCHLENGTH_MIN", ZSTD_SEARCHLENGTH_MIN); + PyModule_AddIntConstant(mod, "SEARCHLENGTH_MAX", ZSTD_SEARCHLENGTH_MAX); + PyModule_AddIntConstant(mod, "TARGETLENGTH_MIN", ZSTD_TARGETLENGTH_MIN); + PyModule_AddIntConstant(mod, "TARGETLENGTH_MAX", ZSTD_TARGETLENGTH_MAX); + + PyModule_AddIntConstant(mod, "STRATEGY_FAST", ZSTD_fast); + PyModule_AddIntConstant(mod, "STRATEGY_DFAST", ZSTD_dfast); + PyModule_AddIntConstant(mod, "STRATEGY_GREEDY", ZSTD_greedy); + PyModule_AddIntConstant(mod, "STRATEGY_LAZY", ZSTD_lazy); + PyModule_AddIntConstant(mod, "STRATEGY_LAZY2", ZSTD_lazy2); + PyModule_AddIntConstant(mod, "STRATEGY_BTLAZY2", ZSTD_btlazy2); + PyModule_AddIntConstant(mod, "STRATEGY_BTOPT", ZSTD_btopt); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressionwriter.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressionwriter.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressionwriter.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressionwriter.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,187 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdDecompressionWriter__doc, +"""A context manager used for writing decompressed output.\n" +); + +static void ZstdDecompressionWriter_dealloc(ZstdDecompressionWriter* self) { + Py_XDECREF(self->decompressor); + Py_XDECREF(self->writer); + + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdDecompressionWriter_enter(ZstdDecompressionWriter* self) { + if (self->entered) { + PyErr_SetString(ZstdError, "cannot __enter__ multiple times"); + return NULL; + } + + self->dstream = DStream_from_ZstdDecompressor(self->decompressor); + if (!self->dstream) { + return NULL; + } + + self->entered = 1; + + Py_INCREF(self); + return (PyObject*)self; +} + +static PyObject* ZstdDecompressionWriter_exit(ZstdDecompressionWriter* self, PyObject* args) { + self->entered = 0; + + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + Py_RETURN_FALSE; +} + +static PyObject* ZstdDecompressionWriter_memory_size(ZstdDecompressionWriter* self) { + if (!self->dstream) { + PyErr_SetString(ZstdError, "cannot determine size of inactive decompressor; " + "call when context manager is active"); + return NULL; + } + + return PyLong_FromSize_t(ZSTD_sizeof_DStream(self->dstream)); +} + +static PyObject* ZstdDecompressionWriter_write(ZstdDecompressionWriter* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t zresult = 0; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + PyObject* res; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", &source, &sourceSize)) { +#else + if (!PyArg_ParseTuple(args, "s#", &source, &sourceSize)) { +#endif + return NULL; + } + + if (!self->entered) { + PyErr_SetString(ZstdError, "write must be called from an active context manager"); + return NULL; + } + + output.dst = PyMem_Malloc(self->outSize); + if (!output.dst) { + return PyErr_NoMemory(); + } + output.size = self->outSize; + output.pos = 0; + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + while ((ssize_t)input.pos < sourceSize) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(self->dstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyMem_Free(output.dst); + PyErr_Format(ZstdError, "zstd decompress error: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + res = PyObject_CallMethod(self->writer, "write", "y#", +#else + res = PyObject_CallMethod(self->writer, "write", "s#", +#endif + output.dst, output.pos); + Py_XDECREF(res); + output.pos = 0; + } + } + + PyMem_Free(output.dst); + + /* TODO return bytes written */ + Py_RETURN_NONE; + } + +static PyMethodDef ZstdDecompressionWriter_methods[] = { + { "__enter__", (PyCFunction)ZstdDecompressionWriter_enter, METH_NOARGS, + PyDoc_STR("Enter a decompression context.") }, + { "__exit__", (PyCFunction)ZstdDecompressionWriter_exit, METH_VARARGS, + PyDoc_STR("Exit a decompression context.") }, + { "memory_size", (PyCFunction)ZstdDecompressionWriter_memory_size, METH_NOARGS, + PyDoc_STR("Obtain the memory size in bytes of the underlying decompressor.") }, + { "write", (PyCFunction)ZstdDecompressionWriter_write, METH_VARARGS, + PyDoc_STR("Compress data") }, + { NULL, NULL } +}; + +PyTypeObject ZstdDecompressionWriterType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressionWriter", /* tp_name */ + sizeof(ZstdDecompressionWriter),/* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdDecompressionWriter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdDecompressionWriter__doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + ZstdDecompressionWriter_methods,/* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressionwriter_module_init(PyObject* mod) { + Py_TYPE(&ZstdDecompressionWriterType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressionWriterType) < 0) { + return; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressobj.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressobj.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressobj.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressobj.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,170 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +PyDoc_STRVAR(DecompressionObj__doc__, +"Perform decompression using a standard library compatible API.\n" +); + +static void DecompressionObj_dealloc(ZstdDecompressionObj* self) { + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + Py_XDECREF(self->decompressor); + + PyObject_Del(self); +} + +static PyObject* DecompressionObj_decompress(ZstdDecompressionObj* self, PyObject* args) { + const char* source; + Py_ssize_t sourceSize; + size_t zresult; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + size_t outSize = ZSTD_DStreamOutSize(); + PyObject* result = NULL; + Py_ssize_t resultSize = 0; + + if (self->finished) { + PyErr_SetString(ZstdError, "cannot use a decompressobj multiple times"); + return NULL; + } + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "y#", +#else + if (!PyArg_ParseTuple(args, "s#", +#endif + &source, &sourceSize)) { + return NULL; + } + + input.src = source; + input.size = sourceSize; + input.pos = 0; + + output.dst = PyMem_Malloc(outSize); + if (!output.dst) { + PyErr_NoMemory(); + return NULL; + } + output.size = outSize; + output.pos = 0; + + /* Read input until exhausted. */ + while (input.pos < input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(self->dstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd decompressor error: %s", + ZSTD_getErrorName(zresult)); + result = NULL; + goto finally; + } + + if (0 == zresult) { + self->finished = 1; + } + + if (output.pos) { + if (result) { + resultSize = PyBytes_GET_SIZE(result); + if (-1 == _PyBytes_Resize(&result, resultSize + output.pos)) { + goto except; + } + + memcpy(PyBytes_AS_STRING(result) + resultSize, + output.dst, output.pos); + } + else { + result = PyBytes_FromStringAndSize(output.dst, output.pos); + if (!result) { + goto except; + } + } + + output.pos = 0; + } + } + + if (!result) { + result = PyBytes_FromString(""); + } + + goto finally; + +except: + Py_DecRef(result); + result = NULL; + +finally: + PyMem_Free(output.dst); + + return result; +} + +static PyMethodDef DecompressionObj_methods[] = { + { "decompress", (PyCFunction)DecompressionObj_decompress, + METH_VARARGS, PyDoc_STR("decompress data") }, + { NULL, NULL } +}; + +PyTypeObject ZstdDecompressionObjType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressionObj", /* tp_name */ + sizeof(ZstdDecompressionObj), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)DecompressionObj_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + DecompressionObj__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + DecompressionObj_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressobj_module_init(PyObject* module) { + Py_TYPE(&ZstdDecompressionObjType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressionObjType) < 0) { + return; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressor.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressor.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressor.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressor.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,672 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +extern PyObject* ZstdError; + +ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor) { + ZSTD_DStream* dstream; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + + dstream = ZSTD_createDStream(); + if (!dstream) { + PyErr_SetString(ZstdError, "could not create DStream"); + return NULL; + } + + if (decompressor->dict) { + dictData = decompressor->dict->dictData; + dictSize = decompressor->dict->dictSize; + } + + if (dictData) { + zresult = ZSTD_initDStream_usingDict(dstream, dictData, dictSize); + } + else { + zresult = ZSTD_initDStream(dstream); + } + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "could not initialize DStream: %s", + ZSTD_getErrorName(zresult)); + return NULL; + } + + return dstream; +} + +PyDoc_STRVAR(Decompressor__doc__, +"ZstdDecompressor(dict_data=None)\n" +"\n" +"Create an object used to perform Zstandard decompression.\n" +"\n" +"An instance can perform multiple decompression operations." +); + +static int Decompressor_init(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "dict_data", + NULL + }; + + ZstdCompressionDict* dict = NULL; + + self->refdctx = NULL; + self->dict = NULL; + self->ddict = NULL; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "|O!", kwlist, + &ZstdCompressionDictType, &dict)) { + return -1; + } + + /* Instead of creating a ZSTD_DCtx for every decompression operation, + we create an instance at object creation time and recycle it via + ZSTD_copyDCTx() on each use. This means each use is a malloc+memcpy + instead of a malloc+init. */ + /* TODO lazily initialize the reference ZSTD_DCtx on first use since + not instances of ZstdDecompressor will use a ZSTD_DCtx. */ + self->refdctx = ZSTD_createDCtx(); + if (!self->refdctx) { + PyErr_NoMemory(); + goto except; + } + + if (dict) { + self->dict = dict; + Py_INCREF(dict); + } + + return 0; + +except: + if (self->refdctx) { + ZSTD_freeDCtx(self->refdctx); + self->refdctx = NULL; + } + + return -1; +} + +static void Decompressor_dealloc(ZstdDecompressor* self) { + if (self->refdctx) { + ZSTD_freeDCtx(self->refdctx); + } + + Py_XDECREF(self->dict); + + if (self->ddict) { + ZSTD_freeDDict(self->ddict); + self->ddict = NULL; + } + + PyObject_Del(self); +} + +PyDoc_STRVAR(Decompressor_copy_stream__doc__, + "copy_stream(ifh, ofh[, read_size=default, write_size=default]) -- decompress data between streams\n" + "\n" + "Compressed data will be read from ``ifh``, decompressed, and written to\n" + "``ofh``. ``ifh`` must have a ``read(size)`` method. ``ofh`` must have a\n" + "``write(data)`` method.\n" + "\n" + "The optional ``read_size`` and ``write_size`` arguments control the chunk\n" + "size of data that is ``read()`` and ``write()`` between streams. They default\n" + "to the default input and output sizes of zstd decompressor streams.\n" +); + +static PyObject* Decompressor_copy_stream(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "ifh", + "ofh", + "read_size", + "write_size", + NULL + }; + + PyObject* source; + PyObject* dest; + size_t inSize = ZSTD_DStreamInSize(); + size_t outSize = ZSTD_DStreamOutSize(); + ZSTD_DStream* dstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + Py_ssize_t totalRead = 0; + Py_ssize_t totalWrite = 0; + char* readBuffer; + Py_ssize_t readSize; + PyObject* readResult; + PyObject* res = NULL; + size_t zresult = 0; + PyObject* writeResult; + PyObject* totalReadPy; + PyObject* totalWritePy; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "OO|kk", kwlist, &source, + &dest, &inSize, &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(source, "read")) { + PyErr_SetString(PyExc_ValueError, "first argument must have a read() method"); + return NULL; + } + + if (!PyObject_HasAttrString(dest, "write")) { + PyErr_SetString(PyExc_ValueError, "second argument must have a write() method"); + return NULL; + } + + /* Prevent free on uninitialized memory in finally. */ + output.dst = NULL; + + dstream = DStream_from_ZstdDecompressor(self); + if (!dstream) { + res = NULL; + goto finally; + } + + output.dst = PyMem_Malloc(outSize); + if (!output.dst) { + PyErr_NoMemory(); + res = NULL; + goto finally; + } + output.size = outSize; + output.pos = 0; + + /* Read source stream until EOF */ + while (1) { + readResult = PyObject_CallMethod(source, "read", "n", inSize); + if (!readResult) { + PyErr_SetString(ZstdError, "could not read() from source"); + goto finally; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + + /* If no data was read, we're at EOF. */ + if (0 == readSize) { + break; + } + + totalRead += readSize; + + /* Send data to decompressor */ + input.src = readBuffer; + input.size = readSize; + input.pos = 0; + + while (input.pos < input.size) { + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(dstream, &output, &input); + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "zstd decompressor error: %s", + ZSTD_getErrorName(zresult)); + res = NULL; + goto finally; + } + + if (output.pos) { +#if PY_MAJOR_VERSION >= 3 + writeResult = PyObject_CallMethod(dest, "write", "y#", +#else + writeResult = PyObject_CallMethod(dest, "write", "s#", +#endif + output.dst, output.pos); + + Py_XDECREF(writeResult); + totalWrite += output.pos; + output.pos = 0; + } + } + } + + /* Source stream is exhausted. Finish up. */ + + ZSTD_freeDStream(dstream); + dstream = NULL; + + totalReadPy = PyLong_FromSsize_t(totalRead); + totalWritePy = PyLong_FromSsize_t(totalWrite); + res = PyTuple_Pack(2, totalReadPy, totalWritePy); + Py_DecRef(totalReadPy); + Py_DecRef(totalWritePy); + + finally: + if (output.dst) { + PyMem_Free(output.dst); + } + + if (dstream) { + ZSTD_freeDStream(dstream); + } + + return res; +} + +PyDoc_STRVAR(Decompressor_decompress__doc__, +"decompress(data[, max_output_size=None]) -- Decompress data in its entirety\n" +"\n" +"This method will decompress the entirety of the argument and return the\n" +"result.\n" +"\n" +"The input bytes are expected to contain a full Zstandard frame (something\n" +"compressed with ``ZstdCompressor.compress()`` or similar). If the input does\n" +"not contain a full frame, an exception will be raised.\n" +"\n" +"If the frame header of the compressed data does not contain the content size\n" +"``max_output_size`` must be specified or ``ZstdError`` will be raised. An\n" +"allocation of size ``max_output_size`` will be performed and an attempt will\n" +"be made to perform decompression into that buffer. If the buffer is too\n" +"small or cannot be allocated, ``ZstdError`` will be raised. The buffer will\n" +"be resized if it is too large.\n" +"\n" +"Uncompressed data could be much larger than compressed data. As a result,\n" +"calling this function could result in a very large memory allocation being\n" +"performed to hold the uncompressed data. Therefore it is **highly**\n" +"recommended to use a streaming decompression method instead of this one.\n" +); + +PyObject* Decompressor_decompress(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "data", + "max_output_size", + NULL + }; + + const char* source; + Py_ssize_t sourceSize; + Py_ssize_t maxOutputSize = 0; + unsigned long long decompressedSize; + size_t destCapacity; + PyObject* result = NULL; + ZSTD_DCtx* dctx = NULL; + void* dictData = NULL; + size_t dictSize = 0; + size_t zresult; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "y#|n", kwlist, +#else + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "s#|n", kwlist, +#endif + &source, &sourceSize, &maxOutputSize)) { + return NULL; + } + + dctx = PyMem_Malloc(ZSTD_sizeof_DCtx(self->refdctx)); + if (!dctx) { + PyErr_NoMemory(); + return NULL; + } + + ZSTD_copyDCtx(dctx, self->refdctx); + + if (self->dict) { + dictData = self->dict->dictData; + dictSize = self->dict->dictSize; + } + + if (dictData && !self->ddict) { + Py_BEGIN_ALLOW_THREADS + self->ddict = ZSTD_createDDict(dictData, dictSize); + Py_END_ALLOW_THREADS + + if (!self->ddict) { + PyErr_SetString(ZstdError, "could not create decompression dict"); + goto except; + } + } + + decompressedSize = ZSTD_getDecompressedSize(source, sourceSize); + /* 0 returned if content size not in the zstd frame header */ + if (0 == decompressedSize) { + if (0 == maxOutputSize) { + PyErr_SetString(ZstdError, "input data invalid or missing content size " + "in frame header"); + goto except; + } + else { + result = PyBytes_FromStringAndSize(NULL, maxOutputSize); + destCapacity = maxOutputSize; + } + } + else { + result = PyBytes_FromStringAndSize(NULL, decompressedSize); + destCapacity = decompressedSize; + } + + if (!result) { + goto except; + } + + Py_BEGIN_ALLOW_THREADS + if (self->ddict) { + zresult = ZSTD_decompress_usingDDict(dctx, PyBytes_AsString(result), destCapacity, + source, sourceSize, self->ddict); + } + else { + zresult = ZSTD_decompressDCtx(dctx, PyBytes_AsString(result), destCapacity, source, sourceSize); + } + Py_END_ALLOW_THREADS + + if (ZSTD_isError(zresult)) { + PyErr_Format(ZstdError, "decompression error: %s", ZSTD_getErrorName(zresult)); + goto except; + } + else if (decompressedSize && zresult != decompressedSize) { + PyErr_Format(ZstdError, "decompression error: decompressed %zu bytes; expected %llu", + zresult, decompressedSize); + goto except; + } + else if (zresult < destCapacity) { + if (_PyBytes_Resize(&result, zresult)) { + goto except; + } + } + + goto finally; + +except: + Py_DecRef(result); + result = NULL; + +finally: + if (dctx) { + PyMem_FREE(dctx); + } + + return result; +} + +PyDoc_STRVAR(Decompressor_decompressobj__doc__, +"decompressobj()\n" +"\n" +"Incrementally feed data into a decompressor.\n" +"\n" +"The returned object exposes a ``decompress(data)`` method. This makes it\n" +"compatible with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor`` so that\n" +"callers can swap in the zstd decompressor while using the same API.\n" +); + +static ZstdDecompressionObj* Decompressor_decompressobj(ZstdDecompressor* self) { + ZstdDecompressionObj* result = PyObject_New(ZstdDecompressionObj, &ZstdDecompressionObjType); + if (!result) { + return NULL; + } + + result->dstream = DStream_from_ZstdDecompressor(self); + if (!result->dstream) { + Py_DecRef((PyObject*)result); + return NULL; + } + + result->decompressor = self; + Py_INCREF(result->decompressor); + + result->finished = 0; + + return result; +} + +PyDoc_STRVAR(Decompressor_read_from__doc__, +"read_from(reader[, read_size=default, write_size=default, skip_bytes=0])\n" +"Read compressed data and return an iterator\n" +"\n" +"Returns an iterator of decompressed data chunks produced from reading from\n" +"the ``reader``.\n" +"\n" +"Compressed data will be obtained from ``reader`` by calling the\n" +"``read(size)`` method of it. The source data will be streamed into a\n" +"decompressor. As decompressed data is available, it will be exposed to the\n" +"returned iterator.\n" +"\n" +"Data is ``read()`` in chunks of size ``read_size`` and exposed to the\n" +"iterator in chunks of size ``write_size``. The default values are the input\n" +"and output sizes for a zstd streaming decompressor.\n" +"\n" +"There is also support for skipping the first ``skip_bytes`` of data from\n" +"the source.\n" +); + +static ZstdDecompressorIterator* Decompressor_read_from(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "reader", + "read_size", + "write_size", + "skip_bytes", + NULL + }; + + PyObject* reader; + size_t inSize = ZSTD_DStreamInSize(); + size_t outSize = ZSTD_DStreamOutSize(); + ZstdDecompressorIterator* result; + size_t skipBytes = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|kkk", kwlist, &reader, + &inSize, &outSize, &skipBytes)) { + return NULL; + } + + if (skipBytes >= inSize) { + PyErr_SetString(PyExc_ValueError, + "skip_bytes must be smaller than read_size"); + return NULL; + } + + result = PyObject_New(ZstdDecompressorIterator, &ZstdDecompressorIteratorType); + if (!result) { + return NULL; + } + + result->decompressor = NULL; + result->reader = NULL; + result->buffer = NULL; + result->dstream = NULL; + result->input.src = NULL; + result->output.dst = NULL; + + if (PyObject_HasAttrString(reader, "read")) { + result->reader = reader; + Py_INCREF(result->reader); + } + else if (1 == PyObject_CheckBuffer(reader)) { + /* Object claims it is a buffer. Try to get a handle to it. */ + result->buffer = PyMem_Malloc(sizeof(Py_buffer)); + if (!result->buffer) { + goto except; + } + + memset(result->buffer, 0, sizeof(Py_buffer)); + + if (0 != PyObject_GetBuffer(reader, result->buffer, PyBUF_CONTIG_RO)) { + goto except; + } + + result->bufferOffset = 0; + } + else { + PyErr_SetString(PyExc_ValueError, + "must pass an object with a read() method or conforms to buffer protocol"); + goto except; + } + + result->decompressor = self; + Py_INCREF(result->decompressor); + + result->inSize = inSize; + result->outSize = outSize; + result->skipBytes = skipBytes; + + result->dstream = DStream_from_ZstdDecompressor(self); + if (!result->dstream) { + goto except; + } + + result->input.src = PyMem_Malloc(inSize); + if (!result->input.src) { + PyErr_NoMemory(); + goto except; + } + result->input.size = 0; + result->input.pos = 0; + + result->output.dst = NULL; + result->output.size = 0; + result->output.pos = 0; + + result->readCount = 0; + result->finishedInput = 0; + result->finishedOutput = 0; + + goto finally; + +except: + if (result->reader) { + Py_DECREF(result->reader); + result->reader = NULL; + } + + if (result->buffer) { + PyBuffer_Release(result->buffer); + Py_DECREF(result->buffer); + result->buffer = NULL; + } + + Py_DECREF(result); + result = NULL; + +finally: + + return result; +} + +PyDoc_STRVAR(Decompressor_write_to__doc__, +"Create a context manager to write decompressed data to an object.\n" +"\n" +"The passed object must have a ``write()`` method.\n" +"\n" +"The caller feeds intput data to the object by calling ``write(data)``.\n" +"Decompressed data is written to the argument given as it is decompressed.\n" +"\n" +"An optional ``write_size`` argument defines the size of chunks to\n" +"``write()`` to the writer. It defaults to the default output size for a zstd\n" +"streaming decompressor.\n" +); + +static ZstdDecompressionWriter* Decompressor_write_to(ZstdDecompressor* self, PyObject* args, PyObject* kwargs) { + static char* kwlist[] = { + "writer", + "write_size", + NULL + }; + + PyObject* writer; + size_t outSize = ZSTD_DStreamOutSize(); + ZstdDecompressionWriter* result; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O|k", kwlist, &writer, &outSize)) { + return NULL; + } + + if (!PyObject_HasAttrString(writer, "write")) { + PyErr_SetString(PyExc_ValueError, "must pass an object with a write() method"); + return NULL; + } + + result = PyObject_New(ZstdDecompressionWriter, &ZstdDecompressionWriterType); + if (!result) { + return NULL; + } + + result->decompressor = self; + Py_INCREF(result->decompressor); + + result->writer = writer; + Py_INCREF(result->writer); + + result->outSize = outSize; + + result->entered = 0; + result->dstream = NULL; + + return result; +} + +static PyMethodDef Decompressor_methods[] = { + { "copy_stream", (PyCFunction)Decompressor_copy_stream, METH_VARARGS | METH_KEYWORDS, + Decompressor_copy_stream__doc__ }, + { "decompress", (PyCFunction)Decompressor_decompress, METH_VARARGS | METH_KEYWORDS, + Decompressor_decompress__doc__ }, + { "decompressobj", (PyCFunction)Decompressor_decompressobj, METH_NOARGS, + Decompressor_decompressobj__doc__ }, + { "read_from", (PyCFunction)Decompressor_read_from, METH_VARARGS | METH_KEYWORDS, + Decompressor_read_from__doc__ }, + { "write_to", (PyCFunction)Decompressor_write_to, METH_VARARGS | METH_KEYWORDS, + Decompressor_write_to__doc__ }, + { NULL, NULL } +}; + +PyTypeObject ZstdDecompressorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressor", /* tp_name */ + sizeof(ZstdDecompressor), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)Decompressor_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + Decompressor__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + Decompressor_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)Decompressor_init, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressor_module_init(PyObject* mod) { + Py_TYPE(&ZstdDecompressorType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressorType) < 0) { + return; + } + + Py_INCREF((PyObject*)&ZstdDecompressorType); + PyModule_AddObject(mod, "ZstdDecompressor", + (PyObject*)&ZstdDecompressorType); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressoriterator.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressoriterator.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/decompressoriterator.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/decompressoriterator.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,254 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +#define min(a, b) (((a) < (b)) ? (a) : (b)) + +extern PyObject* ZstdError; + +PyDoc_STRVAR(ZstdDecompressorIterator__doc__, +"Represents an iterator of decompressed data.\n" +); + +static void ZstdDecompressorIterator_dealloc(ZstdDecompressorIterator* self) { + Py_XDECREF(self->decompressor); + Py_XDECREF(self->reader); + + if (self->buffer) { + PyBuffer_Release(self->buffer); + PyMem_FREE(self->buffer); + self->buffer = NULL; + } + + if (self->dstream) { + ZSTD_freeDStream(self->dstream); + self->dstream = NULL; + } + + if (self->input.src) { + PyMem_Free((void*)self->input.src); + self->input.src = NULL; + } + + PyObject_Del(self); +} + +static PyObject* ZstdDecompressorIterator_iter(PyObject* self) { + Py_INCREF(self); + return self; +} + +static DecompressorIteratorResult read_decompressor_iterator(ZstdDecompressorIterator* self) { + size_t zresult; + PyObject* chunk; + DecompressorIteratorResult result; + size_t oldInputPos = self->input.pos; + + result.chunk = NULL; + + chunk = PyBytes_FromStringAndSize(NULL, self->outSize); + if (!chunk) { + result.errored = 1; + return result; + } + + self->output.dst = PyBytes_AsString(chunk); + self->output.size = self->outSize; + self->output.pos = 0; + + Py_BEGIN_ALLOW_THREADS + zresult = ZSTD_decompressStream(self->dstream, &self->output, &self->input); + Py_END_ALLOW_THREADS + + /* We're done with the pointer. Nullify to prevent anyone from getting a + handle on a Python object. */ + self->output.dst = NULL; + + if (ZSTD_isError(zresult)) { + Py_DECREF(chunk); + PyErr_Format(ZstdError, "zstd decompress error: %s", + ZSTD_getErrorName(zresult)); + result.errored = 1; + return result; + } + + self->readCount += self->input.pos - oldInputPos; + + /* Frame is fully decoded. Input exhausted and output sitting in buffer. */ + if (0 == zresult) { + self->finishedInput = 1; + self->finishedOutput = 1; + } + + /* If it produced output data, return it. */ + if (self->output.pos) { + if (self->output.pos < self->outSize) { + if (_PyBytes_Resize(&chunk, self->output.pos)) { + result.errored = 1; + return result; + } + } + } + else { + Py_DECREF(chunk); + chunk = NULL; + } + + result.errored = 0; + result.chunk = chunk; + + return result; +} + +static PyObject* ZstdDecompressorIterator_iternext(ZstdDecompressorIterator* self) { + PyObject* readResult = NULL; + char* readBuffer; + Py_ssize_t readSize; + Py_ssize_t bufferRemaining; + DecompressorIteratorResult result; + + if (self->finishedOutput) { + PyErr_SetString(PyExc_StopIteration, "output flushed"); + return NULL; + } + + /* If we have data left in the input, consume it. */ + if (self->input.pos < self->input.size) { + result = read_decompressor_iterator(self); + if (result.chunk || result.errored) { + return result.chunk; + } + + /* Else fall through to get more data from input. */ + } + +read_from_source: + + if (!self->finishedInput) { + if (self->reader) { + readResult = PyObject_CallMethod(self->reader, "read", "I", self->inSize); + if (!readResult) { + return NULL; + } + + PyBytes_AsStringAndSize(readResult, &readBuffer, &readSize); + } + else { + assert(self->buffer && self->buffer->buf); + + /* Only support contiguous C arrays for now */ + assert(self->buffer->strides == NULL && self->buffer->suboffsets == NULL); + assert(self->buffer->itemsize == 1); + + /* TODO avoid memcpy() below */ + readBuffer = (char *)self->buffer->buf + self->bufferOffset; + bufferRemaining = self->buffer->len - self->bufferOffset; + readSize = min(bufferRemaining, (Py_ssize_t)self->inSize); + self->bufferOffset += readSize; + } + + if (readSize) { + if (!self->readCount && self->skipBytes) { + assert(self->skipBytes < self->inSize); + if ((Py_ssize_t)self->skipBytes >= readSize) { + PyErr_SetString(PyExc_ValueError, + "skip_bytes larger than first input chunk; " + "this scenario is currently unsupported"); + Py_DecRef(readResult); + return NULL; + } + + readBuffer = readBuffer + self->skipBytes; + readSize -= self->skipBytes; + } + + /* Copy input into previously allocated buffer because it can live longer + than a single function call and we don't want to keep a ref to a Python + object around. This could be changed... */ + memcpy((void*)self->input.src, readBuffer, readSize); + self->input.size = readSize; + self->input.pos = 0; + } + /* No bytes on first read must mean an empty input stream. */ + else if (!self->readCount) { + self->finishedInput = 1; + self->finishedOutput = 1; + Py_DecRef(readResult); + PyErr_SetString(PyExc_StopIteration, "empty input"); + return NULL; + } + else { + self->finishedInput = 1; + } + + /* We've copied the data managed by memory. Discard the Python object. */ + Py_DecRef(readResult); + } + + result = read_decompressor_iterator(self); + if (result.errored || result.chunk) { + return result.chunk; + } + + /* No new output data. Try again unless we know there is no more data. */ + if (!self->finishedInput) { + goto read_from_source; + } + + PyErr_SetString(PyExc_StopIteration, "input exhausted"); + return NULL; +} + +PyTypeObject ZstdDecompressorIteratorType = { + PyVarObject_HEAD_INIT(NULL, 0) + "zstd.ZstdDecompressorIterator", /* tp_name */ + sizeof(ZstdDecompressorIterator), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)ZstdDecompressorIterator_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ + ZstdDecompressorIterator__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + ZstdDecompressorIterator_iter, /* tp_iter */ + (iternextfunc)ZstdDecompressorIterator_iternext, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ +}; + +void decompressoriterator_module_init(PyObject* mod) { + Py_TYPE(&ZstdDecompressorIteratorType) = &PyType_Type; + if (PyType_Ready(&ZstdDecompressorIteratorType) < 0) { + return; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/dictparams.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/dictparams.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/dictparams.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/dictparams.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,125 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#include "python-zstandard.h" + +PyDoc_STRVAR(DictParameters__doc__, +"DictParameters: low-level control over dictionary generation"); + +static PyObject* DictParameters_new(PyTypeObject* subtype, PyObject* args, PyObject* kwargs) { + DictParametersObject* self; + unsigned selectivityLevel; + int compressionLevel; + unsigned notificationLevel; + unsigned dictID; + + if (!PyArg_ParseTuple(args, "IiII", &selectivityLevel, &compressionLevel, + ¬ificationLevel, &dictID)) { + return NULL; + } + + self = (DictParametersObject*)subtype->tp_alloc(subtype, 1); + if (!self) { + return NULL; + } + + self->selectivityLevel = selectivityLevel; + self->compressionLevel = compressionLevel; + self->notificationLevel = notificationLevel; + self->dictID = dictID; + + return (PyObject*)self; +} + +static void DictParameters_dealloc(PyObject* self) { + PyObject_Del(self); +} + +static Py_ssize_t DictParameters_length(PyObject* self) { + return 4; +} + +static PyObject* DictParameters_item(PyObject* o, Py_ssize_t i) { + DictParametersObject* self = (DictParametersObject*)o; + + switch (i) { + case 0: + return PyLong_FromLong(self->selectivityLevel); + case 1: + return PyLong_FromLong(self->compressionLevel); + case 2: + return PyLong_FromLong(self->notificationLevel); + case 3: + return PyLong_FromLong(self->dictID); + default: + PyErr_SetString(PyExc_IndexError, "index out of range"); + return NULL; + } +} + +static PySequenceMethods DictParameters_sq = { + DictParameters_length, /* sq_length */ + 0, /* sq_concat */ + 0, /* sq_repeat */ + DictParameters_item, /* sq_item */ + 0, /* sq_ass_item */ + 0, /* sq_contains */ + 0, /* sq_inplace_concat */ + 0 /* sq_inplace_repeat */ +}; + +PyTypeObject DictParametersType = { + PyVarObject_HEAD_INIT(NULL, 0) + "DictParameters", /* tp_name */ + sizeof(DictParametersObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)DictParameters_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + 0, /* tp_repr */ + 0, /* tp_as_number */ + &DictParameters_sq, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + DictParameters__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + DictParameters_new, /* tp_new */ +}; + +void dictparams_module_init(PyObject* mod) { + Py_TYPE(&DictParametersType) = &PyType_Type; + if (PyType_Ready(&DictParametersType) < 0) { + return; + } + + Py_IncRef((PyObject*)&DictParametersType); + PyModule_AddObject(mod, "DictParameters", (PyObject*)&DictParametersType); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/python-zstandard.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/python-zstandard.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/c-ext/python-zstandard.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/c-ext/python-zstandard.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,178 @@ +/** +* Copyright (c) 2016-present, Gregory Szorc +* All rights reserved. +* +* This software may be modified and distributed under the terms +* of the BSD license. See the LICENSE file for details. +*/ + +#define PY_SSIZE_T_CLEAN +#include + +#define ZSTD_STATIC_LINKING_ONLY +#define ZDICT_STATIC_LINKING_ONLY +#include "mem.h" +#include "zstd.h" +#include "zdict.h" + +#define PYTHON_ZSTANDARD_VERSION "0.6.0" + +typedef enum { + compressorobj_flush_finish, + compressorobj_flush_block, +} CompressorObj_Flush; + +typedef struct { + PyObject_HEAD + unsigned windowLog; + unsigned chainLog; + unsigned hashLog; + unsigned searchLog; + unsigned searchLength; + unsigned targetLength; + ZSTD_strategy strategy; +} CompressionParametersObject; + +extern PyTypeObject CompressionParametersType; + +typedef struct { + PyObject_HEAD + unsigned selectivityLevel; + int compressionLevel; + unsigned notificationLevel; + unsigned dictID; +} DictParametersObject; + +extern PyTypeObject DictParametersType; + +typedef struct { + PyObject_HEAD + + void* dictData; + size_t dictSize; +} ZstdCompressionDict; + +extern PyTypeObject ZstdCompressionDictType; + +typedef struct { + PyObject_HEAD + + int compressionLevel; + ZstdCompressionDict* dict; + ZSTD_CCtx* cctx; + ZSTD_CDict* cdict; + CompressionParametersObject* cparams; + ZSTD_frameParameters fparams; +} ZstdCompressor; + +extern PyTypeObject ZstdCompressorType; + +typedef struct { + PyObject_HEAD + + ZstdCompressor* compressor; + ZSTD_CStream* cstream; + ZSTD_outBuffer output; + int finished; +} ZstdCompressionObj; + +extern PyTypeObject ZstdCompressionObjType; + +typedef struct { + PyObject_HEAD + + ZstdCompressor* compressor; + PyObject* writer; + Py_ssize_t sourceSize; + size_t outSize; + ZSTD_CStream* cstream; + int entered; +} ZstdCompressionWriter; + +extern PyTypeObject ZstdCompressionWriterType; + +typedef struct { + PyObject_HEAD + + ZstdCompressor* compressor; + PyObject* reader; + Py_buffer* buffer; + Py_ssize_t bufferOffset; + Py_ssize_t sourceSize; + size_t inSize; + size_t outSize; + + ZSTD_CStream* cstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + int finishedOutput; + int finishedInput; + PyObject* readResult; +} ZstdCompressorIterator; + +extern PyTypeObject ZstdCompressorIteratorType; + +typedef struct { + PyObject_HEAD + + ZSTD_DCtx* refdctx; + + ZstdCompressionDict* dict; + ZSTD_DDict* ddict; +} ZstdDecompressor; + +extern PyTypeObject ZstdDecompressorType; + +typedef struct { + PyObject_HEAD + + ZstdDecompressor* decompressor; + ZSTD_DStream* dstream; + int finished; +} ZstdDecompressionObj; + +extern PyTypeObject ZstdDecompressionObjType; + +typedef struct { + PyObject_HEAD + + ZstdDecompressor* decompressor; + PyObject* writer; + size_t outSize; + ZSTD_DStream* dstream; + int entered; +} ZstdDecompressionWriter; + +extern PyTypeObject ZstdDecompressionWriterType; + +typedef struct { + PyObject_HEAD + + ZstdDecompressor* decompressor; + PyObject* reader; + Py_buffer* buffer; + Py_ssize_t bufferOffset; + size_t inSize; + size_t outSize; + size_t skipBytes; + ZSTD_DStream* dstream; + ZSTD_inBuffer input; + ZSTD_outBuffer output; + Py_ssize_t readCount; + int finishedInput; + int finishedOutput; +} ZstdDecompressorIterator; + +extern PyTypeObject ZstdDecompressorIteratorType; + +typedef struct { + int errored; + PyObject* chunk; +} DecompressorIteratorResult; + +void ztopy_compression_parameters(CompressionParametersObject* params, ZSTD_compressionParameters* zparams); +CompressionParametersObject* get_compression_parameters(PyObject* self, PyObject* args); +PyObject* estimate_compression_context_size(PyObject* self, PyObject* args); +ZSTD_CStream* CStream_from_ZstdCompressor(ZstdCompressor* compressor, Py_ssize_t sourceSize); +ZSTD_DStream* DStream_from_ZstdDecompressor(ZstdDecompressor* decompressor); +ZstdCompressionDict* train_dictionary(PyObject* self, PyObject* args, PyObject* kwargs); diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/LICENSE mercurial-4.1~rc~trusty1/contrib/python-zstandard/LICENSE --- mercurial-4.0.2~trusty1/contrib/python-zstandard/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/LICENSE 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,27 @@ +Copyright (c) 2016, Gregory Szorc +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its contributors +may be used to endorse or promote products derived from this software without +specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/make_cffi.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/make_cffi.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/make_cffi.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/make_cffi.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,108 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +from __future__ import absolute_import + +import cffi +import distutils.ccompiler +import os +import subprocess +import tempfile + + +HERE = os.path.abspath(os.path.dirname(__file__)) + +SOURCES = ['zstd/%s' % p for p in ( + 'common/entropy_common.c', + 'common/error_private.c', + 'common/fse_decompress.c', + 'common/xxhash.c', + 'common/zstd_common.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'compress/zstd_compress.c', + 'decompress/huf_decompress.c', + 'decompress/zstd_decompress.c', + 'dictBuilder/divsufsort.c', + 'dictBuilder/zdict.c', +)] + +INCLUDE_DIRS = [os.path.join(HERE, d) for d in ( + 'zstd', + 'zstd/common', + 'zstd/compress', + 'zstd/decompress', + 'zstd/dictBuilder', +)] + +# cffi can't parse some of the primitives in zstd.h. So we invoke the +# preprocessor and feed its output into cffi. +compiler = distutils.ccompiler.new_compiler() + +# Needed for MSVC. +if hasattr(compiler, 'initialize'): + compiler.initialize() + +# Distutils doesn't set compiler.preprocessor, so invoke the preprocessor +# manually. +if compiler.compiler_type == 'unix': + args = list(compiler.executables['compiler']) + args.extend([ + '-E', + '-DZSTD_STATIC_LINKING_ONLY', + ]) +elif compiler.compiler_type == 'msvc': + args = [compiler.cc] + args.extend([ + '/EP', + '/DZSTD_STATIC_LINKING_ONLY', + ]) +else: + raise Exception('unsupported compiler type: %s' % compiler.compiler_type) + +# zstd.h includes , which is also included by cffi's boilerplate. +# This can lead to duplicate declarations. So we strip this include from the +# preprocessor invocation. + +with open(os.path.join(HERE, 'zstd', 'zstd.h'), 'rb') as fh: + lines = [l for l in fh if not l.startswith(b'#include ')] + +fd, input_file = tempfile.mkstemp(suffix='.h') +os.write(fd, b''.join(lines)) +os.close(fd) + +args.append(input_file) + +try: + process = subprocess.Popen(args, stdout=subprocess.PIPE) + output = process.communicate()[0] + ret = process.poll() + if ret: + raise Exception('preprocessor exited with error') +finally: + os.unlink(input_file) + +def normalize_output(): + lines = [] + for line in output.splitlines(): + # CFFI's parser doesn't like __attribute__ on UNIX compilers. + if line.startswith(b'__attribute__ ((visibility ("default"))) '): + line = line[len(b'__attribute__ ((visibility ("default"))) '):] + + lines.append(line) + + return b'\n'.join(lines) + +ffi = cffi.FFI() +ffi.set_source('_zstd_cffi', ''' +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +''', sources=SOURCES, include_dirs=INCLUDE_DIRS) + +ffi.cdef(normalize_output().decode('latin1')) + +if __name__ == '__main__': + ffi.compile() diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/MANIFEST.in mercurial-4.1~rc~trusty1/contrib/python-zstandard/MANIFEST.in --- mercurial-4.0.2~trusty1/contrib/python-zstandard/MANIFEST.in 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/MANIFEST.in 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,5 @@ +graft c-ext +graft zstd +include make_cffi.py +include setup_zstd.py +include zstd.c diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/NEWS.rst mercurial-4.1~rc~trusty1/contrib/python-zstandard/NEWS.rst --- mercurial-4.0.2~trusty1/contrib/python-zstandard/NEWS.rst 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/NEWS.rst 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,90 @@ +Version History +=============== + +0.6.0 (released 2017-01-14) +--------------------------- + +* Support for legacy zstd protocols (build time opt in feature). +* Automation improvements to test against Python 3.6, latest versions + of Tox, more deterministic AppVeyor behavior. +* CFFI "parser" improved to use a compiler preprocessor instead of rewriting + source code manually. +* Vendored version of zstd updated to 1.1.2. +* Documentation improvements. +* Introduce a bench.py script for performing (crude) benchmarks. +* ZSTD_CCtx instances are now reused across multiple compress() operations. +* ZstdCompressor.write_to() now has a flush() method. +* ZstdCompressor.compressobj()'s flush() method now accepts an argument to + flush a block (as opposed to ending the stream). +* Disallow compress(b'') when writing content sizes by default (issue #11). + +0.5.2 (released 2016-11-12) +--------------------------- + +* more packaging fixes for source distribution + +0.5.1 (released 2016-11-12) +--------------------------- + +* setup_zstd.py is included in the source distribution + +0.5.0 (released 2016-11-10) +--------------------------- + +* Vendored version of zstd updated to 1.1.1. +* Continuous integration for Python 3.6 and 3.7 +* Continuous integration for Conda +* Added compression and decompression APIs providing similar interfaces + to the standard library ``zlib`` and ``bz2`` modules. This allows + coding to a common interface. +* ``zstd.__version__` is now defined. +* ``read_from()`` on various APIs now accepts objects implementing the buffer + protocol. +* ``read_from()`` has gained a ``skip_bytes`` argument. This allows callers + to pass in an existing buffer with a header without having to create a + slice or a new object. +* Implemented ``ZstdCompressionDict.as_bytes()``. +* Python's memory allocator is now used instead of ``malloc()``. +* Low-level zstd data structures are reused in more instances, cutting down + on overhead for certain operations. +* ``distutils`` boilerplate for obtaining an ``Extension`` instance + has now been refactored into a standalone ``setup_zstd.py`` file. This + allows other projects with ``setup.py`` files to reuse the + ``distutils`` code for this project without copying code. +* The monolithic ``zstd.c`` file has been split into a header file defining + types and separate ``.c`` source files for the implementation. + +History of the Project +====================== + +2016-08-31 - Zstandard 1.0.0 is released and Gregory starts hacking on a +Python extension for use by the Mercurial project. A very hacky prototype +is sent to the mercurial-devel list for RFC. + +2016-09-03 - Most functionality from Zstandard C API implemented. Source +code published on https://github.com/indygreg/python-zstandard. Travis-CI +automation configured. 0.0.1 release on PyPI. + +2016-09-05 - After the API was rounded out a bit and support for Python +2.6 and 2.7 was added, version 0.1 was released to PyPI. + +2016-09-05 - After the compressor and decompressor APIs were changed, 0.2 +was released to PyPI. + +2016-09-10 - 0.3 is released with a bunch of new features. ZstdCompressor +now accepts arguments controlling frame parameters. The source size can now +be declared when performing streaming compression. ZstdDecompressor.decompress() +is implemented. Compression dictionaries are now cached when using the simple +compression and decompression APIs. Memory size APIs added. +ZstdCompressor.read_from() and ZstdDecompressor.read_from() have been +implemented. This rounds out the major compression/decompression APIs planned +by the author. + +2016-10-02 - 0.3.3 is released with a bug fix for read_from not fully +decoding a zstd frame (issue #2). + +2016-10-02 - 0.4.0 is released with zstd 1.1.0, support for custom read and +write buffer sizes, and a few bug fixes involving failure to read/write +all data when buffer sizes were too small to hold remaining data. + +2016-11-10 - 0.5.0 is released with zstd 1.1.1 and other enhancements. diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/README.rst mercurial-4.1~rc~trusty1/contrib/python-zstandard/README.rst --- mercurial-4.0.2~trusty1/contrib/python-zstandard/README.rst 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/README.rst 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,829 @@ +================ +python-zstandard +================ + +This project provides Python bindings for interfacing with the +`Zstandard `_ compression library. A C extension +and CFFI interface is provided. + +The primary goal of the extension is to provide a Pythonic interface to +the underlying C API. This means exposing most of the features and flexibility +of the C API while not sacrificing usability or safety that Python provides. + +The canonical home for this project is +https://github.com/indygreg/python-zstandard. + +| |ci-status| |win-ci-status| + +State of Project +================ + +The project is officially in beta state. The author is reasonably satisfied +with the current API and that functionality works as advertised. There +may be some backwards incompatible changes before 1.0. Though the author +does not intend to make any major changes to the Python API. + +There is continuous integration for Python versions 2.6, 2.7, and 3.3+ +on Linux x86_x64 and Windows x86 and x86_64. The author is reasonably +confident the extension is stable and works as advertised on these +platforms. + +Expected Changes +---------------- + +The author is reasonably confident in the current state of what's +implemented on the ``ZstdCompressor`` and ``ZstdDecompressor`` types. +Those APIs likely won't change significantly. Some low-level behavior +(such as naming and types expected by arguments) may change. + +There will likely be arguments added to control the input and output +buffer sizes (currently, certain operations read and write in chunk +sizes using zstd's preferred defaults). + +There should be an API that accepts an object that conforms to the buffer +interface and returns an iterator over compressed or decompressed output. + +The author is on the fence as to whether to support the extremely +low level compression and decompression APIs. It could be useful to +support compression without the framing headers. But the author doesn't +believe it a high priority at this time. + +The CFFI bindings are half-baked and need to be finished. + +Requirements +============ + +This extension is designed to run with Python 2.6, 2.7, 3.3, 3.4, and 3.5 +on common platforms (Linux, Windows, and OS X). Only x86_64 is currently +well-tested as an architecture. + +Installing +========== + +This package is uploaded to PyPI at https://pypi.python.org/pypi/zstandard. +So, to install this package:: + + $ pip install zstandard + +Binary wheels are made available for some platforms. If you need to +install from a source distribution, all you should need is a working C +compiler and the Python development headers/libraries. On many Linux +distributions, you can install a ``python-dev`` or ``python-devel`` +package to provide these dependencies. + +Packages are also uploaded to Anaconda Cloud at +https://anaconda.org/indygreg/zstandard. See that URL for how to install +this package with ``conda``. + +Performance +=========== + +Very crude and non-scientific benchmarking (most benchmarks fall in this +category because proper benchmarking is hard) show that the Python bindings +perform within 10% of the native C implementation. + +The following table compares the performance of compressing and decompressing +a 1.1 GB tar file comprised of the files in a Firefox source checkout. Values +obtained with the ``zstd`` program are on the left. The remaining columns detail +performance of various compression APIs in the Python bindings. + ++-------+-----------------+-----------------+-----------------+---------------+ +| Level | Native | Simple | Stream In | Stream Out | +| | Comp / Decomp | Comp / Decomp | Comp / Decomp | Comp | ++=======+=================+=================+=================+===============+ +| 1 | 490 / 1338 MB/s | 458 / 1266 MB/s | 407 / 1156 MB/s | 405 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ +| 2 | 412 / 1288 MB/s | 381 / 1203 MB/s | 345 / 1128 MB/s | 349 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ +| 3 | 342 / 1312 MB/s | 319 / 1182 MB/s | 285 / 1165 MB/s | 287 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ +| 11 | 64 / 1506 MB/s | 66 / 1436 MB/s | 56 / 1342 MB/s | 57 MB/s | ++-------+-----------------+-----------------+-----------------+---------------+ + +Again, these are very unscientific. But it shows that Python is capable of +compressing at several hundred MB/s and decompressing at over 1 GB/s. + +Comparison to Other Python Bindings +=================================== + +https://pypi.python.org/pypi/zstd is an alternative Python binding to +Zstandard. At the time this was written, the latest release of that +package (1.0.0.2) had the following significant differences from this package: + +* It only exposes the simple API for compression and decompression operations. + This extension exposes the streaming API, dictionary training, and more. +* It adds a custom framing header to compressed data and there is no way to + disable it. This means that data produced with that module cannot be used by + other Zstandard implementations. + +Bundling of Zstandard Source Code +================================= + +The source repository for this project contains a vendored copy of the +Zstandard source code. This is done for a few reasons. + +First, Zstandard is relatively new and not yet widely available as a system +package. Providing a copy of the source code enables the Python C extension +to be compiled without requiring the user to obtain the Zstandard source code +separately. + +Second, Zstandard has both a stable *public* API and an *experimental* API. +The *experimental* API is actually quite useful (contains functionality for +training dictionaries for example), so it is something we wish to expose to +Python. However, the *experimental* API is only available via static linking. +Furthermore, the *experimental* API can change at any time. So, control over +the exact version of the Zstandard library linked against is important to +ensure known behavior. + +Instructions for Building and Testing +===================================== + +Once you have the source code, the extension can be built via setup.py:: + + $ python setup.py build_ext + +We recommend testing with ``nose``:: + + $ nosetests + +A Tox configuration is present to test against multiple Python versions:: + + $ tox + +Tests use the ``hypothesis`` Python package to perform fuzzing. If you +don't have it, those tests won't run. + +There is also an experimental CFFI module. You need the ``cffi`` Python +package installed to build and test that. + +To create a virtualenv with all development dependencies, do something +like the following:: + + # Python 2 + $ virtualenv venv + + # Python 3 + $ python3 -m venv venv + + $ source venv/bin/activate + $ pip install cffi hypothesis nose tox + +API +=== + +The compiled C extension provides a ``zstd`` Python module. This module +exposes the following interfaces. + +ZstdCompressor +-------------- + +The ``ZstdCompressor`` class provides an interface for performing +compression operations. + +Each instance is associated with parameters that control compression +behavior. These come from the following named arguments (all optional): + +level + Integer compression level. Valid values are between 1 and 22. +dict_data + Compression dictionary to use. + + Note: When using dictionary data and ``compress()`` is called multiple + times, the ``CompressionParameters`` derived from an integer compression + ``level`` and the first compressed data's size will be reused for all + subsequent operations. This may not be desirable if source data size + varies significantly. +compression_params + A ``CompressionParameters`` instance (overrides the ``level`` value). +write_checksum + Whether a 4 byte checksum should be written with the compressed data. + Defaults to False. If True, the decompressor can verify that decompressed + data matches the original input data. +write_content_size + Whether the size of the uncompressed data will be written into the + header of compressed data. Defaults to False. The data will only be + written if the compressor knows the size of the input data. This is + likely not true for streaming compression. +write_dict_id + Whether to write the dictionary ID into the compressed data. + Defaults to True. The dictionary ID is only written if a dictionary + is being used. + +Unless specified otherwise, assume that no two methods of ``ZstdCompressor`` +instances can be called from multiple Python threads simultaneously. In other +words, assume instances are not thread safe unless stated otherwise. + +Simple API +^^^^^^^^^^ + +``compress(data)`` compresses and returns data as a one-shot operation.:: + + cctx = zstd.ZstdCompressor() + compressed = cctx.compress(b'data to compress') + +Unless ``compression_params`` or ``dict_data`` are passed to the +``ZstdCompressor``, each invocation of ``compress()`` will calculate the +optimal compression parameters for the configured compression ``level`` and +input data size (some parameters are fine-tuned for small input sizes). + +If a compression dictionary is being used, the compression parameters +determined from the first input's size will be reused for subsequent +operations. + +There is currently a deficiency in zstd's C APIs that makes it difficult +to round trip empty inputs when ``write_content_size=True``. Attempting +this will raise a ``ValueError`` unless ``allow_empty=True`` is passed +to ``compress()``. + +Streaming Input API +^^^^^^^^^^^^^^^^^^^ + +``write_to(fh)`` (which behaves as a context manager) allows you to *stream* +data into a compressor.:: + + cctx = zstd.ZstdCompressor(level=10) + with cctx.write_to(fh) as compressor: + compressor.write(b'chunk 0') + compressor.write(b'chunk 1') + ... + +The argument to ``write_to()`` must have a ``write(data)`` method. As +compressed data is available, ``write()`` will be called with the compressed +data as its argument. Many common Python types implement ``write()``, including +open file handles and ``io.BytesIO``. + +``write_to()`` returns an object representing a streaming compressor instance. +It **must** be used as a context manager. That object's ``write(data)`` method +is used to feed data into the compressor. + +A ``flush()`` method can be called to evict whatever data remains within the +compressor's internal state into the output object. This may result in 0 or +more ``write()`` calls to the output object. + +If the size of the data being fed to this streaming compressor is known, +you can declare it before compression begins:: + + cctx = zstd.ZstdCompressor() + with cctx.write_to(fh, size=data_len) as compressor: + compressor.write(chunk0) + compressor.write(chunk1) + ... + +Declaring the size of the source data allows compression parameters to +be tuned. And if ``write_content_size`` is used, it also results in the +content size being written into the frame header of the output data. + +The size of chunks being ``write()`` to the destination can be specified:: + + cctx = zstd.ZstdCompressor() + with cctx.write_to(fh, write_size=32768) as compressor: + ... + +To see how much memory is being used by the streaming compressor:: + + cctx = zstd.ZstdCompressor() + with cctx.write_to(fh) as compressor: + ... + byte_size = compressor.memory_size() + +Streaming Output API +^^^^^^^^^^^^^^^^^^^^ + +``read_from(reader)`` provides a mechanism to stream data out of a compressor +as an iterator of data chunks.:: + + cctx = zstd.ZstdCompressor() + for chunk in cctx.read_from(fh): + # Do something with emitted data. + +``read_from()`` accepts an object that has a ``read(size)`` method or conforms +to the buffer protocol. (``bytes`` and ``memoryview`` are 2 common types that +provide the buffer protocol.) + +Uncompressed data is fetched from the source either by calling ``read(size)`` +or by fetching a slice of data from the object directly (in the case where +the buffer protocol is being used). The returned iterator consists of chunks +of compressed data. + +If reading from the source via ``read()``, ``read()`` will be called until +it raises or returns an empty bytes (``b''``). It is perfectly valid for +the source to deliver fewer bytes than were what requested by ``read(size)``. + +Like ``write_to()``, ``read_from()`` also accepts a ``size`` argument +declaring the size of the input stream:: + + cctx = zstd.ZstdCompressor() + for chunk in cctx.read_from(fh, size=some_int): + pass + +You can also control the size that data is ``read()`` from the source and +the ideal size of output chunks:: + + cctx = zstd.ZstdCompressor() + for chunk in cctx.read_from(fh, read_size=16384, write_size=8192): + pass + +Unlike ``write_to()``, ``read_from()`` does not give direct control over the +sizes of chunks fed into the compressor. Instead, chunk sizes will be whatever +the object being read from delivers. These will often be of a uniform size. + +Stream Copying API +^^^^^^^^^^^^^^^^^^ + +``copy_stream(ifh, ofh)`` can be used to copy data between 2 streams while +compressing it.:: + + cctx = zstd.ZstdCompressor() + cctx.copy_stream(ifh, ofh) + +For example, say you wish to compress a file:: + + cctx = zstd.ZstdCompressor() + with open(input_path, 'rb') as ifh, open(output_path, 'wb') as ofh: + cctx.copy_stream(ifh, ofh) + +It is also possible to declare the size of the source stream:: + + cctx = zstd.ZstdCompressor() + cctx.copy_stream(ifh, ofh, size=len_of_input) + +You can also specify how large the chunks that are ``read()`` and ``write()`` +from and to the streams:: + + cctx = zstd.ZstdCompressor() + cctx.copy_stream(ifh, ofh, read_size=32768, write_size=16384) + +The stream copier returns a 2-tuple of bytes read and written:: + + cctx = zstd.ZstdCompressor() + read_count, write_count = cctx.copy_stream(ifh, ofh) + +Compressor API +^^^^^^^^^^^^^^ + +``compressobj()`` returns an object that exposes ``compress(data)`` and +``flush()`` methods. Each returns compressed data or an empty bytes. + +The purpose of ``compressobj()`` is to provide an API-compatible interface +with ``zlib.compressobj`` and ``bz2.BZ2Compressor``. This allows callers to +swap in different compressor objects while using the same API. + +``flush()`` accepts an optional argument indicating how to end the stream. +``zstd.COMPRESSOBJ_FLUSH_FINISH`` (the default) ends the compression stream. +Once this type of flush is performed, ``compress()`` and ``flush()`` can +no longer be called. This type of flush **must** be called to end the +compression context. If not called, returned data may be incomplete. + +A ``zstd.COMPRESSOBJ_FLUSH_BLOCK`` argument to ``flush()`` will flush a +zstd block. Flushes of this type can be performed multiple times. The next +call to ``compress()`` will begin a new zstd block. + +Here is how this API should be used:: + + cctx = zstd.ZstdCompressor() + cobj = cctx.compressobj() + data = cobj.compress(b'raw input 0') + data = cobj.compress(b'raw input 1') + data = cobj.flush() + +Or to flush blocks:: + + cctx.zstd.ZstdCompressor() + cobj = cctx.compressobj() + data = cobj.compress(b'chunk in first block') + data = cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK) + data = cobj.compress(b'chunk in second block') + data = cobj.flush() + +For best performance results, keep input chunks under 256KB. This avoids +extra allocations for a large output object. + +It is possible to declare the input size of the data that will be fed into +the compressor:: + + cctx = zstd.ZstdCompressor() + cobj = cctx.compressobj(size=6) + data = cobj.compress(b'foobar') + data = cobj.flush() + +ZstdDecompressor +---------------- + +The ``ZstdDecompressor`` class provides an interface for performing +decompression. + +Each instance is associated with parameters that control decompression. These +come from the following named arguments (all optional): + +dict_data + Compression dictionary to use. + +The interface of this class is very similar to ``ZstdCompressor`` (by design). + +Unless specified otherwise, assume that no two methods of ``ZstdDecompressor`` +instances can be called from multiple Python threads simultaneously. In other +words, assume instances are not thread safe unless stated otherwise. + +Simple API +^^^^^^^^^^ + +``decompress(data)`` can be used to decompress an entire compressed zstd +frame in a single operation.:: + + dctx = zstd.ZstdDecompressor() + decompressed = dctx.decompress(data) + +By default, ``decompress(data)`` will only work on data written with the content +size encoded in its header. This can be achieved by creating a +``ZstdCompressor`` with ``write_content_size=True``. If compressed data without +an embedded content size is seen, ``zstd.ZstdError`` will be raised. + +If the compressed data doesn't have its content size embedded within it, +decompression can be attempted by specifying the ``max_output_size`` +argument.:: + + dctx = zstd.ZstdDecompressor() + uncompressed = dctx.decompress(data, max_output_size=1048576) + +Ideally, ``max_output_size`` will be identical to the decompressed output +size. + +If ``max_output_size`` is too small to hold the decompressed data, +``zstd.ZstdError`` will be raised. + +If ``max_output_size`` is larger than the decompressed data, the allocated +output buffer will be resized to only use the space required. + +Please note that an allocation of the requested ``max_output_size`` will be +performed every time the method is called. Setting to a very large value could +result in a lot of work for the memory allocator and may result in +``MemoryError`` being raised if the allocation fails. + +If the exact size of decompressed data is unknown, it is **strongly** +recommended to use a streaming API. + +Streaming Input API +^^^^^^^^^^^^^^^^^^^ + +``write_to(fh)`` can be used to incrementally send compressed data to a +decompressor.:: + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(fh) as decompressor: + decompressor.write(compressed_data) + +This behaves similarly to ``zstd.ZstdCompressor``: compressed data is written to +the decompressor by calling ``write(data)`` and decompressed output is written +to the output object by calling its ``write(data)`` method. + +The size of chunks being ``write()`` to the destination can be specified:: + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(fh, write_size=16384) as decompressor: + pass + +You can see how much memory is being used by the decompressor:: + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(fh) as decompressor: + byte_size = decompressor.memory_size() + +Streaming Output API +^^^^^^^^^^^^^^^^^^^^ + +``read_from(fh)`` provides a mechanism to stream decompressed data out of a +compressed source as an iterator of data chunks.:: + + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(fh): + # Do something with original data. + +``read_from()`` accepts a) an object with a ``read(size)`` method that will +return compressed bytes b) an object conforming to the buffer protocol that +can expose its data as a contiguous range of bytes. The ``bytes`` and +``memoryview`` types expose this buffer protocol. + +``read_from()`` returns an iterator whose elements are chunks of the +decompressed data. + +The size of requested ``read()`` from the source can be specified:: + + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(fh, read_size=16384): + pass + +It is also possible to skip leading bytes in the input data:: + + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(fh, skip_bytes=1): + pass + +Skipping leading bytes is useful if the source data contains extra +*header* data but you want to avoid the overhead of making a buffer copy +or allocating a new ``memoryview`` object in order to decompress the data. + +Similarly to ``ZstdCompressor.read_from()``, the consumer of the iterator +controls when data is decompressed. If the iterator isn't consumed, +decompression is put on hold. + +When ``read_from()`` is passed an object conforming to the buffer protocol, +the behavior may seem similar to what occurs when the simple decompression +API is used. However, this API works when the decompressed size is unknown. +Furthermore, if feeding large inputs, the decompressor will work in chunks +instead of performing a single operation. + +Stream Copying API +^^^^^^^^^^^^^^^^^^ + +``copy_stream(ifh, ofh)`` can be used to copy data across 2 streams while +performing decompression.:: + + dctx = zstd.ZstdDecompressor() + dctx.copy_stream(ifh, ofh) + +e.g. to decompress a file to another file:: + + dctx = zstd.ZstdDecompressor() + with open(input_path, 'rb') as ifh, open(output_path, 'wb') as ofh: + dctx.copy_stream(ifh, ofh) + +The size of chunks being ``read()`` and ``write()`` from and to the streams +can be specified:: + + dctx = zstd.ZstdDecompressor() + dctx.copy_stream(ifh, ofh, read_size=8192, write_size=16384) + +Decompressor API +^^^^^^^^^^^^^^^^ + +``decompressobj()`` returns an object that exposes a ``decompress(data)`` +methods. Compressed data chunks are fed into ``decompress(data)`` and +uncompressed output (or an empty bytes) is returned. Output from subsequent +calls needs to be concatenated to reassemble the full decompressed byte +sequence. + +The purpose of ``decompressobj()`` is to provide an API-compatible interface +with ``zlib.decompressobj`` and ``bz2.BZ2Decompressor``. This allows callers +to swap in different decompressor objects while using the same API. + +Each object is single use: once an input frame is decoded, ``decompress()`` +can no longer be called. + +Here is how this API should be used:: + + dctx = zstd.ZstdDeompressor() + dobj = cctx.decompressobj() + data = dobj.decompress(compressed_chunk_0) + data = dobj.decompress(compressed_chunk_1) + +Choosing an API +--------------- + +Various forms of compression and decompression APIs are provided because each +are suitable for different use cases. + +The simple/one-shot APIs are useful for small data, when the decompressed +data size is known (either recorded in the zstd frame header via +``write_content_size`` or known via an out-of-band mechanism, such as a file +size). + +A limitation of the simple APIs is that input or output data must fit in memory. +And unless using advanced tricks with Python *buffer objects*, both input and +output must fit in memory simultaneously. + +Another limitation is that compression or decompression is performed as a single +operation. So if you feed large input, it could take a long time for the +function to return. + +The streaming APIs do not have the limitations of the simple API. The cost to +this is they are more complex to use than a single function call. + +The streaming APIs put the caller in control of compression and decompression +behavior by allowing them to directly control either the input or output side +of the operation. + +With the streaming input APIs, the caller feeds data into the compressor or +decompressor as they see fit. Output data will only be written after the caller +has explicitly written data. + +With the streaming output APIs, the caller consumes output from the compressor +or decompressor as they see fit. The compressor or decompressor will only +consume data from the source when the caller is ready to receive it. + +One end of the streaming APIs involves a file-like object that must +``write()`` output data or ``read()`` input data. Depending on what the +backing storage for these objects is, those operations may not complete quickly. +For example, when streaming compressed data to a file, the ``write()`` into +a streaming compressor could result in a ``write()`` to the filesystem, which +may take a long time to finish due to slow I/O on the filesystem. So, there +may be overhead in streaming APIs beyond the compression and decompression +operations. + +Dictionary Creation and Management +---------------------------------- + +Zstandard allows *dictionaries* to be used when compressing and +decompressing data. The idea is that if you are compressing a lot of similar +data, you can precompute common properties of that data (such as recurring +byte sequences) to achieve better compression ratios. + +In Python, compression dictionaries are represented as the +``ZstdCompressionDict`` type. + +Instances can be constructed from bytes:: + + dict_data = zstd.ZstdCompressionDict(data) + +More interestingly, instances can be created by *training* on sample data:: + + dict_data = zstd.train_dictionary(size, samples) + +This takes a list of bytes instances and creates and returns a +``ZstdCompressionDict``. + +You can see how many bytes are in the dictionary by calling ``len()``:: + + dict_data = zstd.train_dictionary(size, samples) + dict_size = len(dict_data) # will not be larger than ``size`` + +Once you have a dictionary, you can pass it to the objects performing +compression and decompression:: + + dict_data = zstd.train_dictionary(16384, samples) + + cctx = zstd.ZstdCompressor(dict_data=dict_data) + for source_data in input_data: + compressed = cctx.compress(source_data) + # Do something with compressed data. + + dctx = zstd.ZstdDecompressor(dict_data=dict_data) + for compressed_data in input_data: + buffer = io.BytesIO() + with dctx.write_to(buffer) as decompressor: + decompressor.write(compressed_data) + # Do something with raw data in ``buffer``. + +Dictionaries have unique integer IDs. You can retrieve this ID via:: + + dict_id = zstd.dictionary_id(dict_data) + +You can obtain the raw data in the dict (useful for persisting and constructing +a ``ZstdCompressionDict`` later) via ``as_bytes()``:: + + dict_data = zstd.train_dictionary(size, samples) + raw_data = dict_data.as_bytes() + +Explicit Compression Parameters +------------------------------- + +Zstandard's integer compression levels along with the input size and dictionary +size are converted into a data structure defining multiple parameters to tune +behavior of the compression algorithm. It is possible to use define this +data structure explicitly to have lower-level control over compression behavior. + +The ``zstd.CompressionParameters`` type represents this data structure. +You can see how Zstandard converts compression levels to this data structure +by calling ``zstd.get_compression_parameters()``. e.g.:: + + params = zstd.get_compression_parameters(5) + +This function also accepts the uncompressed data size and dictionary size +to adjust parameters:: + + params = zstd.get_compression_parameters(3, source_size=len(data), dict_size=len(dict_data)) + +You can also construct compression parameters from their low-level components:: + + params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST) + +You can then configure a compressor to use the custom parameters:: + + cctx = zstd.ZstdCompressor(compression_params=params) + +The members of the ``CompressionParameters`` tuple are as follows:: + +* 0 - Window log +* 1 - Chain log +* 2 - Hash log +* 3 - Search log +* 4 - Search length +* 5 - Target length +* 6 - Strategy (one of the ``zstd.STRATEGY_`` constants) + +You'll need to read the Zstandard documentation for what these parameters +do. + +Misc Functionality +------------------ + +estimate_compression_context_size(CompressionParameters) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Given a ``CompressionParameters`` struct, estimate the memory size required +to perform compression. + +estimate_decompression_context_size() +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Estimate the memory size requirements for a decompressor instance. + +Constants +--------- + +The following module constants/attributes are exposed: + +ZSTD_VERSION + This module attribute exposes a 3-tuple of the Zstandard version. e.g. + ``(1, 0, 0)`` +MAX_COMPRESSION_LEVEL + Integer max compression level accepted by compression functions +COMPRESSION_RECOMMENDED_INPUT_SIZE + Recommended chunk size to feed to compressor functions +COMPRESSION_RECOMMENDED_OUTPUT_SIZE + Recommended chunk size for compression output +DECOMPRESSION_RECOMMENDED_INPUT_SIZE + Recommended chunk size to feed into decompresor functions +DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE + Recommended chunk size for decompression output + +FRAME_HEADER + bytes containing header of the Zstandard frame +MAGIC_NUMBER + Frame header as an integer + +WINDOWLOG_MIN + Minimum value for compression parameter +WINDOWLOG_MAX + Maximum value for compression parameter +CHAINLOG_MIN + Minimum value for compression parameter +CHAINLOG_MAX + Maximum value for compression parameter +HASHLOG_MIN + Minimum value for compression parameter +HASHLOG_MAX + Maximum value for compression parameter +SEARCHLOG_MIN + Minimum value for compression parameter +SEARCHLOG_MAX + Maximum value for compression parameter +SEARCHLENGTH_MIN + Minimum value for compression parameter +SEARCHLENGTH_MAX + Maximum value for compression parameter +TARGETLENGTH_MIN + Minimum value for compression parameter +TARGETLENGTH_MAX + Maximum value for compression parameter +STRATEGY_FAST + Compression strategory +STRATEGY_DFAST + Compression strategory +STRATEGY_GREEDY + Compression strategory +STRATEGY_LAZY + Compression strategory +STRATEGY_LAZY2 + Compression strategory +STRATEGY_BTLAZY2 + Compression strategory +STRATEGY_BTOPT + Compression strategory + +Note on Zstandard's *Experimental* API +====================================== + +Many of the Zstandard APIs used by this module are marked as *experimental* +within the Zstandard project. This includes a large number of useful +features, such as compression and frame parameters and parts of dictionary +compression. + +It is unclear how Zstandard's C API will evolve over time, especially with +regards to this *experimental* functionality. We will try to maintain +backwards compatibility at the Python API level. However, we cannot +guarantee this for things not under our control. + +Since a copy of the Zstandard source code is distributed with this +module and since we compile against it, the behavior of a specific +version of this module should be constant for all of time. So if you +pin the version of this module used in your projects (which is a Python +best practice), you should be buffered from unwanted future changes. + +Donate +====== + +A lot of time has been invested into this project by the author. + +If you find this project useful and would like to thank the author for +their work, consider donating some money. Any amount is appreciated. + +.. image:: https://www.paypalobjects.com/en_US/i/btn/btn_donate_LG.gif + :target: https://www.paypal.com/cgi-bin/webscr?cmd=_donations&business=gregory%2eszorc%40gmail%2ecom&lc=US&item_name=python%2dzstandard¤cy_code=USD&bn=PP%2dDonationsBF%3abtn_donate_LG%2egif%3aNonHosted + :alt: Donate via PayPal + +.. |ci-status| image:: https://travis-ci.org/indygreg/python-zstandard.svg?branch=master + :target: https://travis-ci.org/indygreg/python-zstandard + +.. |win-ci-status| image:: https://ci.appveyor.com/api/projects/status/github/indygreg/python-zstandard?svg=true + :target: https://ci.appveyor.com/project/indygreg/python-zstandard + :alt: Windows build status diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/setup.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/setup.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/setup.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/setup.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import sys +from setuptools import setup + +try: + import cffi +except ImportError: + cffi = None + +import setup_zstd + +SUPPORT_LEGACY = False + +if "--legacy" in sys.argv: + SUPPORT_LEGACY = True + sys.argv.remove("--legacy") + +# Code for obtaining the Extension instance is in its own module to +# facilitate reuse in other projects. +extensions = [setup_zstd.get_c_extension(SUPPORT_LEGACY, 'zstd')] + +if cffi: + import make_cffi + extensions.append(make_cffi.ffi.distutils_extension()) + +version = None + +with open('c-ext/python-zstandard.h', 'r') as fh: + for line in fh: + if not line.startswith('#define PYTHON_ZSTANDARD_VERSION'): + continue + + version = line.split()[2][1:-1] + break + +if not version: + raise Exception('could not resolve package version; ' + 'this should never happen') + +setup( + name='zstandard', + version=version, + description='Zstandard bindings for Python', + long_description=open('README.rst', 'r').read(), + url='https://github.com/indygreg/python-zstandard', + author='Gregory Szorc', + author_email='gregory.szorc@gmail.com', + license='BSD', + classifiers=[ + 'Development Status :: 4 - Beta', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: C', + 'Programming Language :: Python :: 2.6', + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.3', + 'Programming Language :: Python :: 3.4', + 'Programming Language :: Python :: 3.5', + ], + keywords='zstandard zstd compression', + ext_modules=extensions, + test_suite='tests', +) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/setup_zstd.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/setup_zstd.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/setup_zstd.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/setup_zstd.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,91 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +import os +from distutils.extension import Extension + + +zstd_sources = ['zstd/%s' % p for p in ( + 'common/entropy_common.c', + 'common/error_private.c', + 'common/fse_decompress.c', + 'common/xxhash.c', + 'common/zstd_common.c', + 'compress/fse_compress.c', + 'compress/huf_compress.c', + 'compress/zstd_compress.c', + 'decompress/huf_decompress.c', + 'decompress/zstd_decompress.c', + 'dictBuilder/divsufsort.c', + 'dictBuilder/zdict.c', +)] + +zstd_sources_legacy = ['zstd/%s' % p for p in ( + 'deprecated/zbuff_compress.c', + 'deprecated/zbuff_decompress.c', + 'legacy/zstd_v01.c', + 'legacy/zstd_v02.c', + 'legacy/zstd_v03.c', + 'legacy/zstd_v04.c', + 'legacy/zstd_v05.c', + 'legacy/zstd_v06.c', + 'legacy/zstd_v07.c' +)] + +zstd_includes = [ + 'c-ext', + 'zstd', + 'zstd/common', + 'zstd/compress', + 'zstd/decompress', + 'zstd/dictBuilder', +] + +zstd_includes_legacy = [ + 'zstd/deprecated', + 'zstd/legacy', +] + +ext_sources = [ + 'zstd.c', + 'c-ext/compressiondict.c', + 'c-ext/compressobj.c', + 'c-ext/compressor.c', + 'c-ext/compressoriterator.c', + 'c-ext/compressionparams.c', + 'c-ext/compressionwriter.c', + 'c-ext/constants.c', + 'c-ext/decompressobj.c', + 'c-ext/decompressor.c', + 'c-ext/decompressoriterator.c', + 'c-ext/decompressionwriter.c', + 'c-ext/dictparams.c', +] + +zstd_depends = [ + 'c-ext/python-zstandard.h', +] + + +def get_c_extension(support_legacy=False, name='zstd'): + """Obtain a distutils.extension.Extension for the C extension.""" + root = os.path.abspath(os.path.dirname(__file__)) + + sources = [os.path.join(root, p) for p in zstd_sources + ext_sources] + if support_legacy: + sources.extend([os.path.join(root, p) for p in zstd_sources_legacy]) + + include_dirs = [os.path.join(root, d) for d in zstd_includes] + if support_legacy: + include_dirs.extend([os.path.join(root, d) for d in zstd_includes_legacy]) + + depends = [os.path.join(root, p) for p in zstd_depends] + + # TODO compile with optimizations. + return Extension(name, sources, + include_dirs=include_dirs, + depends=depends, + extra_compile_args=["-DZSTD_LEGACY_SUPPORT=1"] if support_legacy else []) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/common.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/common.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/common.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/common.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,15 @@ +import io + +class OpCountingBytesIO(io.BytesIO): + def __init__(self, *args, **kwargs): + self._read_count = 0 + self._write_count = 0 + return super(OpCountingBytesIO, self).__init__(*args, **kwargs) + + def read(self, *args): + self._read_count += 1 + return super(OpCountingBytesIO, self).read(*args) + + def write(self, data): + self._write_count += 1 + return super(OpCountingBytesIO, self).write(data) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_cffi.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_cffi.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_cffi.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_cffi.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,35 @@ +import io + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +try: + import zstd_cffi +except ImportError: + raise unittest.SkipTest('cffi version of zstd not available') + + +class TestCFFIWriteToToCDecompressor(unittest.TestCase): + def test_simple(self): + orig = io.BytesIO() + orig.write(b'foo') + orig.write(b'bar') + orig.write(b'foobar' * 16384) + + dest = io.BytesIO() + cctx = zstd_cffi.ZstdCompressor() + with cctx.write_to(dest) as compressor: + compressor.write(orig.getvalue()) + + uncompressed = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(uncompressed) as decompressor: + decompressor.write(dest.getvalue()) + + self.assertEqual(uncompressed.getvalue(), orig.getvalue()) + + diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_compressor.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_compressor.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_compressor.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_compressor.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,536 @@ +import hashlib +import io +import struct +import sys + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +from .common import OpCountingBytesIO + + +if sys.version_info[0] >= 3: + next = lambda it: it.__next__() +else: + next = lambda it: it.next() + + +class TestCompressor(unittest.TestCase): + def test_level_bounds(self): + with self.assertRaises(ValueError): + zstd.ZstdCompressor(level=0) + + with self.assertRaises(ValueError): + zstd.ZstdCompressor(level=23) + + +class TestCompressor_compress(unittest.TestCase): + def test_compress_empty(self): + cctx = zstd.ZstdCompressor(level=1) + cctx.compress(b'') + + cctx = zstd.ZstdCompressor(level=22) + cctx.compress(b'') + + def test_compress_empty(self): + cctx = zstd.ZstdCompressor(level=1) + self.assertEqual(cctx.compress(b''), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + # TODO should be temporary until https://github.com/facebook/zstd/issues/506 + # is fixed. + cctx = zstd.ZstdCompressor(write_content_size=True) + with self.assertRaises(ValueError): + cctx.compress(b'') + + cctx.compress(b'', allow_empty=True) + + def test_compress_large(self): + chunks = [] + for i in range(255): + chunks.append(struct.Struct('>B').pack(i) * 16384) + + cctx = zstd.ZstdCompressor(level=3) + result = cctx.compress(b''.join(chunks)) + self.assertEqual(len(result), 999) + self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd') + + def test_write_checksum(self): + cctx = zstd.ZstdCompressor(level=1) + no_checksum = cctx.compress(b'foobar') + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + with_checksum = cctx.compress(b'foobar') + + self.assertEqual(len(with_checksum), len(no_checksum) + 4) + + def test_write_content_size(self): + cctx = zstd.ZstdCompressor(level=1) + no_size = cctx.compress(b'foobar' * 256) + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + with_size = cctx.compress(b'foobar' * 256) + + self.assertEqual(len(with_size), len(no_size) + 1) + + def test_no_dict_id(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(1024, samples) + + cctx = zstd.ZstdCompressor(level=1, dict_data=d) + with_dict_id = cctx.compress(b'foobarfoobar') + + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False) + no_dict_id = cctx.compress(b'foobarfoobar') + + self.assertEqual(len(with_dict_id), len(no_dict_id) + 4) + + def test_compress_dict_multiple(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + cctx = zstd.ZstdCompressor(level=1, dict_data=d) + + for i in range(32): + cctx.compress(b'foo bar foobar foo bar foobar') + + +class TestCompressor_compressobj(unittest.TestCase): + def test_compressobj_empty(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj() + self.assertEqual(cobj.compress(b''), b'') + self.assertEqual(cobj.flush(), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_compressobj_large(self): + chunks = [] + for i in range(255): + chunks.append(struct.Struct('>B').pack(i) * 16384) + + cctx = zstd.ZstdCompressor(level=3) + cobj = cctx.compressobj() + + result = cobj.compress(b''.join(chunks)) + cobj.flush() + self.assertEqual(len(result), 999) + self.assertEqual(result[0:4], b'\x28\xb5\x2f\xfd') + + def test_write_checksum(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj() + no_checksum = cobj.compress(b'foobar') + cobj.flush() + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + cobj = cctx.compressobj() + with_checksum = cobj.compress(b'foobar') + cobj.flush() + + self.assertEqual(len(with_checksum), len(no_checksum) + 4) + + def test_write_content_size(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj(size=len(b'foobar' * 256)) + no_size = cobj.compress(b'foobar' * 256) + cobj.flush() + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + cobj = cctx.compressobj(size=len(b'foobar' * 256)) + with_size = cobj.compress(b'foobar' * 256) + cobj.flush() + + self.assertEqual(len(with_size), len(no_size) + 1) + + def test_compress_after_finished(self): + cctx = zstd.ZstdCompressor() + cobj = cctx.compressobj() + + cobj.compress(b'foo') + cobj.flush() + + with self.assertRaisesRegexp(zstd.ZstdError, 'cannot call compress\(\) after compressor'): + cobj.compress(b'foo') + + with self.assertRaisesRegexp(zstd.ZstdError, 'compressor object already finished'): + cobj.flush() + + def test_flush_block_repeated(self): + cctx = zstd.ZstdCompressor(level=1) + cobj = cctx.compressobj() + + self.assertEqual(cobj.compress(b'foo'), b'') + self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), + b'\x28\xb5\x2f\xfd\x00\x48\x18\x00\x00foo') + self.assertEqual(cobj.compress(b'bar'), b'') + # 3 byte header plus content. + self.assertEqual(cobj.flush(), b'\x19\x00\x00bar') + + def test_flush_empty_block(self): + cctx = zstd.ZstdCompressor(write_checksum=True) + cobj = cctx.compressobj() + + cobj.compress(b'foobar') + cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK) + # No-op if no block is active (this is internal to zstd). + self.assertEqual(cobj.flush(zstd.COMPRESSOBJ_FLUSH_BLOCK), b'') + + trailing = cobj.flush() + # 3 bytes block header + 4 bytes frame checksum + self.assertEqual(len(trailing), 7) + header = trailing[0:3] + self.assertEqual(header, b'\x01\x00\x00') + + +class TestCompressor_copy_stream(unittest.TestCase): + def test_no_read(self): + source = object() + dest = io.BytesIO() + + cctx = zstd.ZstdCompressor() + with self.assertRaises(ValueError): + cctx.copy_stream(source, dest) + + def test_no_write(self): + source = io.BytesIO() + dest = object() + + cctx = zstd.ZstdCompressor() + with self.assertRaises(ValueError): + cctx.copy_stream(source, dest) + + def test_empty(self): + source = io.BytesIO() + dest = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=1) + r, w = cctx.copy_stream(source, dest) + self.assertEqual(int(r), 0) + self.assertEqual(w, 9) + + self.assertEqual(dest.getvalue(), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_large_data(self): + source = io.BytesIO() + for i in range(255): + source.write(struct.Struct('>B').pack(i) * 16384) + source.seek(0) + + dest = io.BytesIO() + cctx = zstd.ZstdCompressor() + r, w = cctx.copy_stream(source, dest) + + self.assertEqual(r, 255 * 16384) + self.assertEqual(w, 999) + + def test_write_checksum(self): + source = io.BytesIO(b'foobar') + no_checksum = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=1) + cctx.copy_stream(source, no_checksum) + + source.seek(0) + with_checksum = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + cctx.copy_stream(source, with_checksum) + + self.assertEqual(len(with_checksum.getvalue()), + len(no_checksum.getvalue()) + 4) + + def test_write_content_size(self): + source = io.BytesIO(b'foobar' * 256) + no_size = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=1) + cctx.copy_stream(source, no_size) + + source.seek(0) + with_size = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + cctx.copy_stream(source, with_size) + + # Source content size is unknown, so no content size written. + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue())) + + source.seek(0) + with_size = io.BytesIO() + cctx.copy_stream(source, with_size, size=len(source.getvalue())) + + # We specified source size, so content size header is present. + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue()) + 1) + + def test_read_write_size(self): + source = OpCountingBytesIO(b'foobarfoobar') + dest = OpCountingBytesIO() + cctx = zstd.ZstdCompressor() + r, w = cctx.copy_stream(source, dest, read_size=1, write_size=1) + + self.assertEqual(r, len(source.getvalue())) + self.assertEqual(w, 21) + self.assertEqual(source._read_count, len(source.getvalue()) + 1) + self.assertEqual(dest._write_count, len(dest.getvalue())) + + +def compress(data, level): + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(level=level) + with cctx.write_to(buffer) as compressor: + compressor.write(data) + return buffer.getvalue() + + +class TestCompressor_write_to(unittest.TestCase): + def test_empty(self): + self.assertEqual(compress(b'', 1), + b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + def test_multiple_compress(self): + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(level=5) + with cctx.write_to(buffer) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'x' * 8192) + + result = buffer.getvalue() + self.assertEqual(result, + b'\x28\xb5\x2f\xfd\x00\x50\x75\x00\x00\x38\x66\x6f' + b'\x6f\x62\x61\x72\x78\x01\x00\xfc\xdf\x03\x23') + + def test_dictionary(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(level=9, dict_data=d) + with cctx.write_to(buffer) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'foo' * 16384) + + compressed = buffer.getvalue() + h = hashlib.sha1(compressed).hexdigest() + self.assertEqual(h, '1c5bcd25181bcd8c1a73ea8773323e0056129f92') + + def test_compression_params(self): + params = zstd.CompressionParameters(20, 6, 12, 5, 4, 10, zstd.STRATEGY_FAST) + + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(compression_params=params) + with cctx.write_to(buffer) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'foobar' * 16384) + + compressed = buffer.getvalue() + h = hashlib.sha1(compressed).hexdigest() + self.assertEqual(h, '1ae31f270ed7de14235221a604b31ecd517ebd99') + + def test_write_checksum(self): + no_checksum = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1) + with cctx.write_to(no_checksum) as compressor: + compressor.write(b'foobar') + + with_checksum = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_checksum=True) + with cctx.write_to(with_checksum) as compressor: + compressor.write(b'foobar') + + self.assertEqual(len(with_checksum.getvalue()), + len(no_checksum.getvalue()) + 4) + + def test_write_content_size(self): + no_size = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1) + with cctx.write_to(no_size) as compressor: + compressor.write(b'foobar' * 256) + + with_size = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, write_content_size=True) + with cctx.write_to(with_size) as compressor: + compressor.write(b'foobar' * 256) + + # Source size is not known in streaming mode, so header not + # written. + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue())) + + # Declaring size will write the header. + with_size = io.BytesIO() + with cctx.write_to(with_size, size=len(b'foobar' * 256)) as compressor: + compressor.write(b'foobar' * 256) + + self.assertEqual(len(with_size.getvalue()), + len(no_size.getvalue()) + 1) + + def test_no_dict_id(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(1024, samples) + + with_dict_id = io.BytesIO() + cctx = zstd.ZstdCompressor(level=1, dict_data=d) + with cctx.write_to(with_dict_id) as compressor: + compressor.write(b'foobarfoobar') + + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False) + no_dict_id = io.BytesIO() + with cctx.write_to(no_dict_id) as compressor: + compressor.write(b'foobarfoobar') + + self.assertEqual(len(with_dict_id.getvalue()), + len(no_dict_id.getvalue()) + 4) + + def test_memory_size(self): + cctx = zstd.ZstdCompressor(level=3) + buffer = io.BytesIO() + with cctx.write_to(buffer) as compressor: + size = compressor.memory_size() + + self.assertGreater(size, 100000) + + def test_write_size(self): + cctx = zstd.ZstdCompressor(level=3) + dest = OpCountingBytesIO() + with cctx.write_to(dest, write_size=1) as compressor: + compressor.write(b'foo') + compressor.write(b'bar') + compressor.write(b'foobar') + + self.assertEqual(len(dest.getvalue()), dest._write_count) + + def test_flush_repeated(self): + cctx = zstd.ZstdCompressor(level=3) + dest = OpCountingBytesIO() + with cctx.write_to(dest) as compressor: + compressor.write(b'foo') + self.assertEqual(dest._write_count, 0) + compressor.flush() + self.assertEqual(dest._write_count, 1) + compressor.write(b'bar') + self.assertEqual(dest._write_count, 1) + compressor.flush() + self.assertEqual(dest._write_count, 2) + compressor.write(b'baz') + + self.assertEqual(dest._write_count, 3) + + def test_flush_empty_block(self): + cctx = zstd.ZstdCompressor(level=3, write_checksum=True) + dest = OpCountingBytesIO() + with cctx.write_to(dest) as compressor: + compressor.write(b'foobar' * 8192) + count = dest._write_count + offset = dest.tell() + compressor.flush() + self.assertGreater(dest._write_count, count) + self.assertGreater(dest.tell(), offset) + offset = dest.tell() + # Ending the write here should cause an empty block to be written + # to denote end of frame. + + trailing = dest.getvalue()[offset:] + # 3 bytes block header + 4 bytes frame checksum + self.assertEqual(len(trailing), 7) + + header = trailing[0:3] + self.assertEqual(header, b'\x01\x00\x00') + + +class TestCompressor_read_from(unittest.TestCase): + def test_type_validation(self): + cctx = zstd.ZstdCompressor() + + # Object with read() works. + cctx.read_from(io.BytesIO()) + + # Buffer protocol works. + cctx.read_from(b'foobar') + + with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'): + cctx.read_from(True) + + def test_read_empty(self): + cctx = zstd.ZstdCompressor(level=1) + + source = io.BytesIO() + it = cctx.read_from(source) + chunks = list(it) + self.assertEqual(len(chunks), 1) + compressed = b''.join(chunks) + self.assertEqual(compressed, b'\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00') + + # And again with the buffer protocol. + it = cctx.read_from(b'') + chunks = list(it) + self.assertEqual(len(chunks), 1) + compressed2 = b''.join(chunks) + self.assertEqual(compressed2, compressed) + + def test_read_large(self): + cctx = zstd.ZstdCompressor(level=1) + + source = io.BytesIO() + source.write(b'f' * zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE) + source.write(b'o') + source.seek(0) + + # Creating an iterator should not perform any compression until + # first read. + it = cctx.read_from(source, size=len(source.getvalue())) + self.assertEqual(source.tell(), 0) + + # We should have exactly 2 output chunks. + chunks = [] + chunk = next(it) + self.assertIsNotNone(chunk) + self.assertEqual(source.tell(), zstd.COMPRESSION_RECOMMENDED_INPUT_SIZE) + chunks.append(chunk) + chunk = next(it) + self.assertIsNotNone(chunk) + chunks.append(chunk) + + self.assertEqual(source.tell(), len(source.getvalue())) + + with self.assertRaises(StopIteration): + next(it) + + # And again for good measure. + with self.assertRaises(StopIteration): + next(it) + + # We should get the same output as the one-shot compression mechanism. + self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue())) + + # Now check the buffer protocol. + it = cctx.read_from(source.getvalue()) + chunks = list(it) + self.assertEqual(len(chunks), 2) + self.assertEqual(b''.join(chunks), cctx.compress(source.getvalue())) + + def test_read_write_size(self): + source = OpCountingBytesIO(b'foobarfoobar') + cctx = zstd.ZstdCompressor(level=3) + for chunk in cctx.read_from(source, read_size=1, write_size=1): + self.assertEqual(len(chunk), 1) + + self.assertEqual(source._read_count, len(source.getvalue()) + 1) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_data_structures.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_data_structures.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_data_structures.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_data_structures.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,107 @@ +import io + +try: + import unittest2 as unittest +except ImportError: + import unittest + +try: + import hypothesis + import hypothesis.strategies as strategies +except ImportError: + hypothesis = None + +import zstd + +class TestCompressionParameters(unittest.TestCase): + def test_init_bad_arg_type(self): + with self.assertRaises(TypeError): + zstd.CompressionParameters() + + with self.assertRaises(TypeError): + zstd.CompressionParameters(0, 1) + + def test_bounds(self): + zstd.CompressionParameters(zstd.WINDOWLOG_MIN, + zstd.CHAINLOG_MIN, + zstd.HASHLOG_MIN, + zstd.SEARCHLOG_MIN, + zstd.SEARCHLENGTH_MIN, + zstd.TARGETLENGTH_MIN, + zstd.STRATEGY_FAST) + + zstd.CompressionParameters(zstd.WINDOWLOG_MAX, + zstd.CHAINLOG_MAX, + zstd.HASHLOG_MAX, + zstd.SEARCHLOG_MAX, + zstd.SEARCHLENGTH_MAX, + zstd.TARGETLENGTH_MAX, + zstd.STRATEGY_BTOPT) + + def test_get_compression_parameters(self): + p = zstd.get_compression_parameters(1) + self.assertIsInstance(p, zstd.CompressionParameters) + + self.assertEqual(p[0], 19) + +if hypothesis: + s_windowlog = strategies.integers(min_value=zstd.WINDOWLOG_MIN, + max_value=zstd.WINDOWLOG_MAX) + s_chainlog = strategies.integers(min_value=zstd.CHAINLOG_MIN, + max_value=zstd.CHAINLOG_MAX) + s_hashlog = strategies.integers(min_value=zstd.HASHLOG_MIN, + max_value=zstd.HASHLOG_MAX) + s_searchlog = strategies.integers(min_value=zstd.SEARCHLOG_MIN, + max_value=zstd.SEARCHLOG_MAX) + s_searchlength = strategies.integers(min_value=zstd.SEARCHLENGTH_MIN, + max_value=zstd.SEARCHLENGTH_MAX) + s_targetlength = strategies.integers(min_value=zstd.TARGETLENGTH_MIN, + max_value=zstd.TARGETLENGTH_MAX) + s_strategy = strategies.sampled_from((zstd.STRATEGY_FAST, + zstd.STRATEGY_DFAST, + zstd.STRATEGY_GREEDY, + zstd.STRATEGY_LAZY, + zstd.STRATEGY_LAZY2, + zstd.STRATEGY_BTLAZY2, + zstd.STRATEGY_BTOPT)) + + class TestCompressionParametersHypothesis(unittest.TestCase): + @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog, + s_searchlength, s_targetlength, s_strategy) + def test_valid_init(self, windowlog, chainlog, hashlog, searchlog, + searchlength, targetlength, strategy): + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + self.assertEqual(tuple(p), + (windowlog, chainlog, hashlog, searchlog, + searchlength, targetlength, strategy)) + + # Verify we can instantiate a compressor with the supplied values. + # ZSTD_checkCParams moves the goal posts on us from what's advertised + # in the constants. So move along with them. + if searchlength == zstd.SEARCHLENGTH_MIN and strategy in (zstd.STRATEGY_FAST, zstd.STRATEGY_GREEDY): + searchlength += 1 + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + elif searchlength == zstd.SEARCHLENGTH_MAX and strategy != zstd.STRATEGY_FAST: + searchlength -= 1 + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + + cctx = zstd.ZstdCompressor(compression_params=p) + with cctx.write_to(io.BytesIO()): + pass + + @hypothesis.given(s_windowlog, s_chainlog, s_hashlog, s_searchlog, + s_searchlength, s_targetlength, s_strategy) + def test_estimate_compression_context_size(self, windowlog, chainlog, + hashlog, searchlog, + searchlength, targetlength, + strategy): + p = zstd.CompressionParameters(windowlog, chainlog, hashlog, + searchlog, searchlength, + targetlength, strategy) + size = zstd.estimate_compression_context_size(p) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_decompressor.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_decompressor.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_decompressor.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_decompressor.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,478 @@ +import io +import random +import struct +import sys + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +from .common import OpCountingBytesIO + + +if sys.version_info[0] >= 3: + next = lambda it: it.__next__() +else: + next = lambda it: it.next() + + +class TestDecompressor_decompress(unittest.TestCase): + def test_empty_input(self): + dctx = zstd.ZstdDecompressor() + + with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'): + dctx.decompress(b'') + + def test_invalid_input(self): + dctx = zstd.ZstdDecompressor() + + with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'): + dctx.decompress(b'foobar') + + def test_no_content_size_in_frame(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + compressed = cctx.compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + with self.assertRaisesRegexp(zstd.ZstdError, 'input data invalid'): + dctx.decompress(compressed) + + def test_content_size_present(self): + cctx = zstd.ZstdCompressor(write_content_size=True) + compressed = cctx.compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + decompressed = dctx.decompress(compressed) + self.assertEqual(decompressed, b'foobar') + + def test_max_output_size(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + source = b'foobar' * 256 + compressed = cctx.compress(source) + + dctx = zstd.ZstdDecompressor() + # Will fit into buffer exactly the size of input. + decompressed = dctx.decompress(compressed, max_output_size=len(source)) + self.assertEqual(decompressed, source) + + # Input size - 1 fails + with self.assertRaisesRegexp(zstd.ZstdError, 'Destination buffer is too small'): + dctx.decompress(compressed, max_output_size=len(source) - 1) + + # Input size + 1 works + decompressed = dctx.decompress(compressed, max_output_size=len(source) + 1) + self.assertEqual(decompressed, source) + + # A much larger buffer works. + decompressed = dctx.decompress(compressed, max_output_size=len(source) * 64) + self.assertEqual(decompressed, source) + + def test_stupidly_large_output_buffer(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + compressed = cctx.compress(b'foobar' * 256) + dctx = zstd.ZstdDecompressor() + + # Will get OverflowError on some Python distributions that can't + # handle really large integers. + with self.assertRaises((MemoryError, OverflowError)): + dctx.decompress(compressed, max_output_size=2**62) + + def test_dictionary(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + orig = b'foobar' * 16384 + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_content_size=True) + compressed = cctx.compress(orig) + + dctx = zstd.ZstdDecompressor(dict_data=d) + decompressed = dctx.decompress(compressed) + + self.assertEqual(decompressed, orig) + + def test_dictionary_multiple(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + sources = (b'foobar' * 8192, b'foo' * 8192, b'bar' * 8192) + compressed = [] + cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_content_size=True) + for source in sources: + compressed.append(cctx.compress(source)) + + dctx = zstd.ZstdDecompressor(dict_data=d) + for i in range(len(sources)): + decompressed = dctx.decompress(compressed[i]) + self.assertEqual(decompressed, sources[i]) + + +class TestDecompressor_copy_stream(unittest.TestCase): + def test_no_read(self): + source = object() + dest = io.BytesIO() + + dctx = zstd.ZstdDecompressor() + with self.assertRaises(ValueError): + dctx.copy_stream(source, dest) + + def test_no_write(self): + source = io.BytesIO() + dest = object() + + dctx = zstd.ZstdDecompressor() + with self.assertRaises(ValueError): + dctx.copy_stream(source, dest) + + def test_empty(self): + source = io.BytesIO() + dest = io.BytesIO() + + dctx = zstd.ZstdDecompressor() + # TODO should this raise an error? + r, w = dctx.copy_stream(source, dest) + + self.assertEqual(r, 0) + self.assertEqual(w, 0) + self.assertEqual(dest.getvalue(), b'') + + def test_large_data(self): + source = io.BytesIO() + for i in range(255): + source.write(struct.Struct('>B').pack(i) * 16384) + source.seek(0) + + compressed = io.BytesIO() + cctx = zstd.ZstdCompressor() + cctx.copy_stream(source, compressed) + + compressed.seek(0) + dest = io.BytesIO() + dctx = zstd.ZstdDecompressor() + r, w = dctx.copy_stream(compressed, dest) + + self.assertEqual(r, len(compressed.getvalue())) + self.assertEqual(w, len(source.getvalue())) + + def test_read_write_size(self): + source = OpCountingBytesIO(zstd.ZstdCompressor().compress( + b'foobarfoobar')) + + dest = OpCountingBytesIO() + dctx = zstd.ZstdDecompressor() + r, w = dctx.copy_stream(source, dest, read_size=1, write_size=1) + + self.assertEqual(r, len(source.getvalue())) + self.assertEqual(w, len(b'foobarfoobar')) + self.assertEqual(source._read_count, len(source.getvalue()) + 1) + self.assertEqual(dest._write_count, len(dest.getvalue())) + + +class TestDecompressor_decompressobj(unittest.TestCase): + def test_simple(self): + data = zstd.ZstdCompressor(level=1).compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + dobj = dctx.decompressobj() + self.assertEqual(dobj.decompress(data), b'foobar') + + def test_reuse(self): + data = zstd.ZstdCompressor(level=1).compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + dobj = dctx.decompressobj() + dobj.decompress(data) + + with self.assertRaisesRegexp(zstd.ZstdError, 'cannot use a decompressobj'): + dobj.decompress(data) + + +def decompress_via_writer(data): + buffer = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(buffer) as decompressor: + decompressor.write(data) + return buffer.getvalue() + + +class TestDecompressor_write_to(unittest.TestCase): + def test_empty_roundtrip(self): + cctx = zstd.ZstdCompressor() + empty = cctx.compress(b'') + self.assertEqual(decompress_via_writer(empty), b'') + + def test_large_roundtrip(self): + chunks = [] + for i in range(255): + chunks.append(struct.Struct('>B').pack(i) * 16384) + orig = b''.join(chunks) + cctx = zstd.ZstdCompressor() + compressed = cctx.compress(orig) + + self.assertEqual(decompress_via_writer(compressed), orig) + + def test_multiple_calls(self): + chunks = [] + for i in range(255): + for j in range(255): + chunks.append(struct.Struct('>B').pack(j) * i) + + orig = b''.join(chunks) + cctx = zstd.ZstdCompressor() + compressed = cctx.compress(orig) + + buffer = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(buffer) as decompressor: + pos = 0 + while pos < len(compressed): + pos2 = pos + 8192 + decompressor.write(compressed[pos:pos2]) + pos += 8192 + self.assertEqual(buffer.getvalue(), orig) + + def test_dictionary(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + + d = zstd.train_dictionary(8192, samples) + + orig = b'foobar' * 16384 + buffer = io.BytesIO() + cctx = zstd.ZstdCompressor(dict_data=d) + with cctx.write_to(buffer) as compressor: + compressor.write(orig) + + compressed = buffer.getvalue() + buffer = io.BytesIO() + + dctx = zstd.ZstdDecompressor(dict_data=d) + with dctx.write_to(buffer) as decompressor: + decompressor.write(compressed) + + self.assertEqual(buffer.getvalue(), orig) + + def test_memory_size(self): + dctx = zstd.ZstdDecompressor() + buffer = io.BytesIO() + with dctx.write_to(buffer) as decompressor: + size = decompressor.memory_size() + + self.assertGreater(size, 100000) + + def test_write_size(self): + source = zstd.ZstdCompressor().compress(b'foobarfoobar') + dest = OpCountingBytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(dest, write_size=1) as decompressor: + s = struct.Struct('>B') + for c in source: + if not isinstance(c, str): + c = s.pack(c) + decompressor.write(c) + + + self.assertEqual(dest.getvalue(), b'foobarfoobar') + self.assertEqual(dest._write_count, len(dest.getvalue())) + + +class TestDecompressor_read_from(unittest.TestCase): + def test_type_validation(self): + dctx = zstd.ZstdDecompressor() + + # Object with read() works. + dctx.read_from(io.BytesIO()) + + # Buffer protocol works. + dctx.read_from(b'foobar') + + with self.assertRaisesRegexp(ValueError, 'must pass an object with a read'): + dctx.read_from(True) + + def test_empty_input(self): + dctx = zstd.ZstdDecompressor() + + source = io.BytesIO() + it = dctx.read_from(source) + # TODO this is arguably wrong. Should get an error about missing frame foo. + with self.assertRaises(StopIteration): + next(it) + + it = dctx.read_from(b'') + with self.assertRaises(StopIteration): + next(it) + + def test_invalid_input(self): + dctx = zstd.ZstdDecompressor() + + source = io.BytesIO(b'foobar') + it = dctx.read_from(source) + with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'): + next(it) + + it = dctx.read_from(b'foobar') + with self.assertRaisesRegexp(zstd.ZstdError, 'Unknown frame descriptor'): + next(it) + + def test_empty_roundtrip(self): + cctx = zstd.ZstdCompressor(level=1, write_content_size=False) + empty = cctx.compress(b'') + + source = io.BytesIO(empty) + source.seek(0) + + dctx = zstd.ZstdDecompressor() + it = dctx.read_from(source) + + # No chunks should be emitted since there is no data. + with self.assertRaises(StopIteration): + next(it) + + # Again for good measure. + with self.assertRaises(StopIteration): + next(it) + + def test_skip_bytes_too_large(self): + dctx = zstd.ZstdDecompressor() + + with self.assertRaisesRegexp(ValueError, 'skip_bytes must be smaller than read_size'): + dctx.read_from(b'', skip_bytes=1, read_size=1) + + with self.assertRaisesRegexp(ValueError, 'skip_bytes larger than first input chunk'): + b''.join(dctx.read_from(b'foobar', skip_bytes=10)) + + def test_skip_bytes(self): + cctx = zstd.ZstdCompressor(write_content_size=False) + compressed = cctx.compress(b'foobar') + + dctx = zstd.ZstdDecompressor() + output = b''.join(dctx.read_from(b'hdr' + compressed, skip_bytes=3)) + self.assertEqual(output, b'foobar') + + def test_large_output(self): + source = io.BytesIO() + source.write(b'f' * zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE) + source.write(b'o') + source.seek(0) + + cctx = zstd.ZstdCompressor(level=1) + compressed = io.BytesIO(cctx.compress(source.getvalue())) + compressed.seek(0) + + dctx = zstd.ZstdDecompressor() + it = dctx.read_from(compressed) + + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(decompressed, source.getvalue()) + + # And again with buffer protocol. + it = dctx.read_from(compressed.getvalue()) + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(decompressed, source.getvalue()) + + def test_large_input(self): + bytes = list(struct.Struct('>B').pack(i) for i in range(256)) + compressed = io.BytesIO() + input_size = 0 + cctx = zstd.ZstdCompressor(level=1) + with cctx.write_to(compressed) as compressor: + while True: + compressor.write(random.choice(bytes)) + input_size += 1 + + have_compressed = len(compressed.getvalue()) > zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE + have_raw = input_size > zstd.DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE * 2 + if have_compressed and have_raw: + break + + compressed.seek(0) + self.assertGreater(len(compressed.getvalue()), + zstd.DECOMPRESSION_RECOMMENDED_INPUT_SIZE) + + dctx = zstd.ZstdDecompressor() + it = dctx.read_from(compressed) + + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(len(decompressed), input_size) + + # And again with buffer protocol. + it = dctx.read_from(compressed.getvalue()) + + chunks = [] + chunks.append(next(it)) + chunks.append(next(it)) + chunks.append(next(it)) + + with self.assertRaises(StopIteration): + next(it) + + decompressed = b''.join(chunks) + self.assertEqual(len(decompressed), input_size) + + def test_interesting(self): + # Found this edge case via fuzzing. + cctx = zstd.ZstdCompressor(level=1) + + source = io.BytesIO() + + compressed = io.BytesIO() + with cctx.write_to(compressed) as compressor: + for i in range(256): + chunk = b'\0' * 1024 + compressor.write(chunk) + source.write(chunk) + + dctx = zstd.ZstdDecompressor() + + simple = dctx.decompress(compressed.getvalue(), + max_output_size=len(source.getvalue())) + self.assertEqual(simple, source.getvalue()) + + compressed.seek(0) + streamed = b''.join(dctx.read_from(compressed)) + self.assertEqual(streamed, source.getvalue()) + + def test_read_write_size(self): + source = OpCountingBytesIO(zstd.ZstdCompressor().compress(b'foobarfoobar')) + dctx = zstd.ZstdDecompressor() + for chunk in dctx.read_from(source, read_size=1, write_size=1): + self.assertEqual(len(chunk), 1) + + self.assertEqual(source._read_count, len(source.getvalue())) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_estimate_sizes.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_estimate_sizes.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_estimate_sizes.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_estimate_sizes.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,17 @@ +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + + +class TestSizes(unittest.TestCase): + def test_decompression_size(self): + size = zstd.estimate_decompression_context_size() + self.assertGreater(size, 100000) + + def test_compression_size(self): + params = zstd.get_compression_parameters(3) + size = zstd.estimate_compression_context_size(params) + self.assertGreater(size, 100000) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_module_attributes.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_module_attributes.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_module_attributes.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_module_attributes.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,48 @@ +from __future__ import unicode_literals + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + +class TestModuleAttributes(unittest.TestCase): + def test_version(self): + self.assertEqual(zstd.ZSTD_VERSION, (1, 1, 2)) + + def test_constants(self): + self.assertEqual(zstd.MAX_COMPRESSION_LEVEL, 22) + self.assertEqual(zstd.FRAME_HEADER, b'\x28\xb5\x2f\xfd') + + def test_hasattr(self): + attrs = ( + 'COMPRESSION_RECOMMENDED_INPUT_SIZE', + 'COMPRESSION_RECOMMENDED_OUTPUT_SIZE', + 'DECOMPRESSION_RECOMMENDED_INPUT_SIZE', + 'DECOMPRESSION_RECOMMENDED_OUTPUT_SIZE', + 'MAGIC_NUMBER', + 'WINDOWLOG_MIN', + 'WINDOWLOG_MAX', + 'CHAINLOG_MIN', + 'CHAINLOG_MAX', + 'HASHLOG_MIN', + 'HASHLOG_MAX', + 'HASHLOG3_MAX', + 'SEARCHLOG_MIN', + 'SEARCHLOG_MAX', + 'SEARCHLENGTH_MIN', + 'SEARCHLENGTH_MAX', + 'TARGETLENGTH_MIN', + 'TARGETLENGTH_MAX', + 'STRATEGY_FAST', + 'STRATEGY_DFAST', + 'STRATEGY_GREEDY', + 'STRATEGY_LAZY', + 'STRATEGY_LAZY2', + 'STRATEGY_BTLAZY2', + 'STRATEGY_BTOPT', + ) + + for a in attrs: + self.assertTrue(hasattr(zstd, a)) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_roundtrip.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_roundtrip.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_roundtrip.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_roundtrip.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,64 @@ +import io + +try: + import unittest2 as unittest +except ImportError: + import unittest + +try: + import hypothesis + import hypothesis.strategies as strategies +except ImportError: + raise unittest.SkipTest('hypothesis not available') + +import zstd + + +compression_levels = strategies.integers(min_value=1, max_value=22) + + +class TestRoundTrip(unittest.TestCase): + @hypothesis.given(strategies.binary(), compression_levels) + def test_compress_write_to(self, data, level): + """Random data from compress() roundtrips via write_to.""" + cctx = zstd.ZstdCompressor(level=level) + compressed = cctx.compress(data) + + buffer = io.BytesIO() + dctx = zstd.ZstdDecompressor() + with dctx.write_to(buffer) as decompressor: + decompressor.write(compressed) + + self.assertEqual(buffer.getvalue(), data) + + @hypothesis.given(strategies.binary(), compression_levels) + def test_compressor_write_to_decompressor_write_to(self, data, level): + """Random data from compressor write_to roundtrips via write_to.""" + compress_buffer = io.BytesIO() + decompressed_buffer = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=level) + with cctx.write_to(compress_buffer) as compressor: + compressor.write(data) + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(decompressed_buffer) as decompressor: + decompressor.write(compress_buffer.getvalue()) + + self.assertEqual(decompressed_buffer.getvalue(), data) + + @hypothesis.given(strategies.binary(average_size=1048576)) + @hypothesis.settings(perform_health_check=False) + def test_compressor_write_to_decompressor_write_to_larger(self, data): + compress_buffer = io.BytesIO() + decompressed_buffer = io.BytesIO() + + cctx = zstd.ZstdCompressor(level=5) + with cctx.write_to(compress_buffer) as compressor: + compressor.write(data) + + dctx = zstd.ZstdDecompressor() + with dctx.write_to(decompressed_buffer) as decompressor: + decompressor.write(compress_buffer.getvalue()) + + self.assertEqual(decompressed_buffer.getvalue(), data) diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_train_dictionary.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_train_dictionary.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/tests/test_train_dictionary.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/tests/test_train_dictionary.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,46 @@ +import sys + +try: + import unittest2 as unittest +except ImportError: + import unittest + +import zstd + + +if sys.version_info[0] >= 3: + int_type = int +else: + int_type = long + + +class TestTrainDictionary(unittest.TestCase): + def test_no_args(self): + with self.assertRaises(TypeError): + zstd.train_dictionary() + + def test_bad_args(self): + with self.assertRaises(TypeError): + zstd.train_dictionary(8192, u'foo') + + with self.assertRaises(ValueError): + zstd.train_dictionary(8192, [u'foo']) + + def test_basic(self): + samples = [] + for i in range(128): + samples.append(b'foo' * 64) + samples.append(b'bar' * 64) + samples.append(b'foobar' * 64) + samples.append(b'baz' * 64) + samples.append(b'foobaz' * 64) + samples.append(b'bazfoo' * 64) + + d = zstd.train_dictionary(8192, samples) + self.assertLessEqual(len(d), 8192) + + dict_id = d.dict_id() + self.assertIsInstance(dict_id, int_type) + + data = d.as_bytes() + self.assertEqual(data[0:4], b'\x37\xa4\x30\xec') diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/bitstream.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/bitstream.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/bitstream.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/bitstream.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,414 @@ +/* ****************************************************************** + bitstream + Part of FSE library + header file (to include) + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef BITSTREAM_H_MODULE +#define BITSTREAM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* +* This API consists of small unitary functions, which must be inlined for best performance. +* Since link-time-optimization is not available for all compilers, +* these functions are defined into a .h to be included. +*/ + +/*-**************************************** +* Dependencies +******************************************/ +#include "mem.h" /* unaligned access routines */ +#include "error_private.h" /* error codes and messages */ + + +/*========================================= +* Target specific +=========================================*/ +#if defined(__BMI__) && defined(__GNUC__) +# include /* support for bextr (experimental) */ +#endif + + +/*-****************************************** +* bitStream encoding API (write forward) +********************************************/ +/* bitStream can mix input from multiple sources. +* A critical property of these streams is that they encode and decode in **reverse** direction. +* So the first bit sequence you add will be the last to be read, like a LIFO stack. +*/ +typedef struct +{ + size_t bitContainer; + int bitPos; + char* startPtr; + char* ptr; + char* endPtr; +} BIT_CStream_t; + +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* dstBuffer, size_t dstCapacity); +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC); +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC); + +/* Start with initCStream, providing the size of buffer to write into. +* bitStream will never write outside of this buffer. +* `dstCapacity` must be >= sizeof(bitD->bitContainer), otherwise @return will be an error code. +* +* bits are first added to a local register. +* Local register is size_t, hence 64-bits on 64-bits systems, or 32-bits on 32-bits systems. +* Writing data into memory is an explicit operation, performed by the flushBits function. +* Hence keep track how many bits are potentially stored into local register to avoid register overflow. +* After a flushBits, a maximum of 7 bits might still be stored into local register. +* +* Avoid storing elements of more than 24 bits if you want compatibility with 32-bits bitstream readers. +* +* Last operation is to close the bitStream. +* The function returns the final size of CStream in bytes. +* If data couldn't fit into `dstBuffer`, it will return a 0 ( == not storable) +*/ + + +/*-******************************************** +* bitStream decoding API (read backward) +**********************************************/ +typedef struct +{ + size_t bitContainer; + unsigned bitsConsumed; + const char* ptr; + const char* start; +} BIT_DStream_t; + +typedef enum { BIT_DStream_unfinished = 0, + BIT_DStream_endOfBuffer = 1, + BIT_DStream_completed = 2, + BIT_DStream_overflow = 3 } BIT_DStream_status; /* result of BIT_reloadDStream() */ + /* 1,2,4,8 would be better for bitmap combinations, but slows down performance a bit ... :( */ + +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize); +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, unsigned nbBits); +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD); +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* bitD); + + +/* Start by invoking BIT_initDStream(). +* A chunk of the bitStream is then stored into a local register. +* Local register size is 64-bits on 64-bits systems, 32-bits on 32-bits systems (size_t). +* You can then retrieve bitFields stored into the local register, **in reverse order**. +* Local register is explicitly reloaded from memory by the BIT_reloadDStream() method. +* A reload guarantee a minimum of ((8*sizeof(bitD->bitContainer))-7) bits when its result is BIT_DStream_unfinished. +* Otherwise, it can be less than that, so proceed accordingly. +* Checking if DStream has reached its end can be performed with BIT_endOfDStream(). +*/ + + +/*-**************************************** +* unsafe API +******************************************/ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits); +/* faster, but works only if value is "clean", meaning all high bits above nbBits are 0 */ + +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); +/* unsafe version; does not check buffer overflow */ + +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); +/* faster, but works only if nbBits >= 1 */ + + + +/*-************************************************************** +* Internal functions +****************************************************************/ +MEM_STATIC unsigned BIT_highbit32 (register U32 val) +{ +# if defined(_MSC_VER) /* Visual */ + unsigned long r=0; + _BitScanReverse ( &r, val ); + return (unsigned) r; +# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ + return 31 - __builtin_clz (val); +# else /* Software version */ + static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; +# endif +} + +/*===== Local Constants =====*/ +static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F, 0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF, 0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF, 0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF }; /* up to 26 bits */ + + +/*-************************************************************** +* bitStream encoding +****************************************************************/ +/*! BIT_initCStream() : + * `dstCapacity` must be > sizeof(void*) + * @return : 0 if success, + otherwise an error code (can be tested using ERR_isError() ) */ +MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, void* startPtr, size_t dstCapacity) +{ + bitC->bitContainer = 0; + bitC->bitPos = 0; + bitC->startPtr = (char*)startPtr; + bitC->ptr = bitC->startPtr; + bitC->endPtr = bitC->startPtr + dstCapacity - sizeof(bitC->ptr); + if (dstCapacity <= sizeof(bitC->ptr)) return ERROR(dstSize_tooSmall); + return 0; +} + +/*! BIT_addBits() : + can add up to 26 bits into `bitC`. + Does not check for register overflow ! */ +MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_addBitsFast() : + * works only if `value` is _clean_, meaning all high bits above nbBits are 0 */ +MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC, size_t value, unsigned nbBits) +{ + bitC->bitContainer |= value << bitC->bitPos; + bitC->bitPos += nbBits; +} + +/*! BIT_flushBitsFast() : + * unsafe version; does not check buffer overflow */ +MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ +} + +/*! BIT_flushBits() : + * safe version; check for buffer overflow, and prevents it. + * note : does not signal buffer overflow. This will be revealed later on using BIT_closeCStream() */ +MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC) +{ + size_t const nbBytes = bitC->bitPos >> 3; + MEM_writeLEST(bitC->ptr, bitC->bitContainer); + bitC->ptr += nbBytes; + if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr; + bitC->bitPos &= 7; + bitC->bitContainer >>= nbBytes*8; /* if bitPos >= sizeof(bitContainer)*8 --> undefined behavior */ +} + +/*! BIT_closeCStream() : + * @return : size of CStream, in bytes, + or 0 if it could not fit into dstBuffer */ +MEM_STATIC size_t BIT_closeCStream(BIT_CStream_t* bitC) +{ + BIT_addBitsFast(bitC, 1, 1); /* endMark */ + BIT_flushBits(bitC); + + if (bitC->ptr >= bitC->endPtr) return 0; /* doesn't fit within authorized budget : cancel */ + + return (bitC->ptr - bitC->startPtr) + (bitC->bitPos > 0); +} + + +/*-******************************************************** +* bitStream decoding +**********************************************************/ +/*! BIT_initDStream() : +* Initialize a BIT_DStream_t. +* `bitD` : a pointer to an already allocated BIT_DStream_t structure. +* `srcSize` must be the *exact* size of the bitStream, in bytes. +* @return : size of stream (== srcSize) or an errorCode if a problem is detected +*/ +MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, size_t srcSize) +{ + if (srcSize < 1) { memset(bitD, 0, sizeof(*bitD)); return ERROR(srcSize_wrong); } + + if (srcSize >= sizeof(bitD->bitContainer)) { /* normal case */ + bitD->start = (const char*)srcBuffer; + bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); + bitD->bitContainer = MEM_readLEST(bitD->ptr); + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + } else { + bitD->start = (const char*)srcBuffer; + bitD->ptr = bitD->start; + bitD->bitContainer = *(const BYTE*)(bitD->start); + switch(srcSize) + { + case 7: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[6]) << (sizeof(bitD->bitContainer)*8 - 16); + case 6: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[5]) << (sizeof(bitD->bitContainer)*8 - 24); + case 5: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[4]) << (sizeof(bitD->bitContainer)*8 - 32); + case 4: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[3]) << 24; + case 3: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[2]) << 16; + case 2: bitD->bitContainer += (size_t)(((const BYTE*)(srcBuffer))[1]) << 8; + default:; + } + { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; + bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } + bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; + } + + return srcSize; +} + +MEM_STATIC size_t BIT_getUpperBits(size_t bitContainer, U32 const start) +{ + return bitContainer >> start; +} + +MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 const nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) && __GNUC__*1000+__GNUC_MINOR__ >= 4008 /* experimental */ +# if defined(__x86_64__) + if (sizeof(bitContainer)==8) + return _bextr_u64(bitContainer, start, nbBits); + else +# endif + return _bextr_u32(bitContainer, start, nbBits); +#else + return (bitContainer >> start) & BIT_mask[nbBits]; +#endif +} + +MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ + return bitContainer & BIT_mask[nbBits]; +} + +/*! BIT_lookBits() : + * Provides next n bits from local register. + * local register is not modified. + * On 32-bits, maxNbBits==24. + * On 64-bits, maxNbBits==56. + * @return : value extracted + */ + MEM_STATIC size_t BIT_lookBits(const BIT_DStream_t* bitD, U32 nbBits) +{ +#if defined(__BMI__) && defined(__GNUC__) /* experimental; fails if bitD->bitsConsumed + nbBits > sizeof(bitD->bitContainer)*8 */ + return BIT_getMiddleBits(bitD->bitContainer, (sizeof(bitD->bitContainer)*8) - bitD->bitsConsumed - nbBits, nbBits); +#else + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return ((bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> 1) >> ((bitMask-nbBits) & bitMask); +#endif +} + +/*! BIT_lookBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_lookBitsFast(const BIT_DStream_t* bitD, U32 nbBits) +{ + U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; + return (bitD->bitContainer << (bitD->bitsConsumed & bitMask)) >> (((bitMask+1)-nbBits) & bitMask); +} + +MEM_STATIC void BIT_skipBits(BIT_DStream_t* bitD, U32 nbBits) +{ + bitD->bitsConsumed += nbBits; +} + +/*! BIT_readBits() : + * Read (consume) next n bits from local register and update. + * Pay attention to not read more than nbBits contained into local register. + * @return : extracted value. + */ +MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBits(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_readBitsFast() : +* unsafe version; only works only if nbBits >= 1 */ +MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) +{ + size_t const value = BIT_lookBitsFast(bitD, nbBits); + BIT_skipBits(bitD, nbBits); + return value; +} + +/*! BIT_reloadDStream() : +* Refill `bitD` from buffer previously set in BIT_initDStream() . +* This function is safe, it guarantees it will not read beyond src buffer. +* @return : status of `BIT_DStream_t` internal register. + if status == BIT_DStream_unfinished, internal register is filled with >= (sizeof(bitD->bitContainer)*8 - 7) bits */ +MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +{ + if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* should not happen => corruption detected */ + return BIT_DStream_overflow; + + if (bitD->ptr >= bitD->start + sizeof(bitD->bitContainer)) { + bitD->ptr -= bitD->bitsConsumed >> 3; + bitD->bitsConsumed &= 7; + bitD->bitContainer = MEM_readLEST(bitD->ptr); + return BIT_DStream_unfinished; + } + if (bitD->ptr == bitD->start) { + if (bitD->bitsConsumed < sizeof(bitD->bitContainer)*8) return BIT_DStream_endOfBuffer; + return BIT_DStream_completed; + } + { U32 nbBytes = bitD->bitsConsumed >> 3; + BIT_DStream_status result = BIT_DStream_unfinished; + if (bitD->ptr - nbBytes < bitD->start) { + nbBytes = (U32)(bitD->ptr - bitD->start); /* ptr > start */ + result = BIT_DStream_endOfBuffer; + } + bitD->ptr -= nbBytes; + bitD->bitsConsumed -= nbBytes*8; + bitD->bitContainer = MEM_readLEST(bitD->ptr); /* reminder : srcSize > sizeof(bitD) */ + return result; + } +} + +/*! BIT_endOfDStream() : +* @return Tells if DStream has exactly reached its end (all bits consumed). +*/ +MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) +{ + return ((DStream->ptr == DStream->start) && (DStream->bitsConsumed == sizeof(DStream->bitContainer)*8)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* BITSTREAM_H_MODULE */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/entropy_common.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/entropy_common.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/entropy_common.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/entropy_common.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,227 @@ +/* + Common functions of New Generation Entropy library + Copyright (C) 2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +*************************************************************************** */ + +/* ************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" /* ERR_*, ERROR */ +#define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ +#include "huf.h" + + +/*-**************************************** +* FSE Error Management +******************************************/ +unsigned FSE_isError(size_t code) { return ERR_isError(code); } + +const char* FSE_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/* ************************************************************** +* HUF Error Management +****************************************************************/ +unsigned HUF_isError(size_t code) { return ERR_isError(code); } + +const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } + + +/*-************************************************************** +* FSE NCount encoding-decoding +****************************************************************/ +static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } + +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, + const void* headerBuffer, size_t hbSize) +{ + const BYTE* const istart = (const BYTE*) headerBuffer; + const BYTE* const iend = istart + hbSize; + const BYTE* ip = istart; + int nbBits; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + if (hbSize < 4) return ERROR(srcSize_wrong); + bitStream = MEM_readLE32(ip); + nbBits = (bitStream & 0xF) + FSE_MIN_TABLELOG; /* extract tableLog */ + if (nbBits > FSE_TABLELOG_ABSOLUTE_MAX) return ERROR(tableLog_tooLarge); + bitStream >>= 4; + bitCount = 4; + *tableLogPtr = nbBits; + remaining = (1<1) & (charnum<=*maxSVPtr)) { + if (previous0) { + unsigned n0 = charnum; + while ((bitStream & 0xFFFF) == 0xFFFF) { + n0 += 24; + if (ip < iend-5) { + ip += 2; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 16; + bitCount += 16; + } } + while ((bitStream & 3) == 3) { + n0 += 3; + bitStream >>= 2; + bitCount += 2; + } + n0 += bitStream & 3; + bitCount += 2; + if (n0 > *maxSVPtr) return ERROR(maxSymbolValue_tooSmall); + while (charnum < n0) normalizedCounter[charnum++] = 0; + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + bitStream = MEM_readLE32(ip) >> bitCount; + } else { + bitStream >>= 2; + } } + { short const max = (short)((2*threshold-1)-remaining); + short count; + + if ((bitStream & (threshold-1)) < (U32)max) { + count = (short)(bitStream & (threshold-1)); + bitCount += nbBits-1; + } else { + count = (short)(bitStream & (2*threshold-1)); + if (count >= threshold) count -= max; + bitCount += nbBits; + } + + count--; /* extra accuracy */ + remaining -= FSE_abs(count); + normalizedCounter[charnum++] = count; + previous0 = !count; + while (remaining < threshold) { + nbBits--; + threshold >>= 1; + } + + if ((ip <= iend-7) || (ip + (bitCount>>3) <= iend-4)) { + ip += bitCount>>3; + bitCount &= 7; + } else { + bitCount -= (int)(8 * (iend - 4 - ip)); + ip = iend - 4; + } + bitStream = MEM_readLE32(ip) >> (bitCount & 31); + } } /* while ((remaining>1) & (charnum<=*maxSVPtr)) */ + if (remaining != 1) return ERROR(corruption_detected); + if (bitCount > 32) return ERROR(corruption_detected); + *maxSVPtr = charnum-1; + + ip += (bitCount+7)>>3; + return ip-istart; +} + + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + `rankStats` is assumed to be a table of at least HUF_TABLELOG_MAX U32. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableX?() . +*/ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize) +{ + U32 weightTotal; + const BYTE* ip = (const BYTE*) src; + size_t iSize; + size_t oSize; + + if (!srcSize) return ERROR(srcSize_wrong); + iSize = ip[0]; + /* memset(huffWeight, 0, hwSize); *//* is not necessary, even though some analyzer complain ... */ + + if (iSize >= 128) { /* special header */ + oSize = iSize - 127; + iSize = ((oSize+1)/2); + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + if (oSize >= hwSize) return ERROR(corruption_detected); + ip += 1; + { U32 n; + for (n=0; n> 4; + huffWeight[n+1] = ip[n/2] & 15; + } } } + else { /* header compressed with FSE (normal case) */ + FSE_DTable fseWorkspace[FSE_DTABLE_SIZE_U32(6)]; /* 6 is max possible tableLog for HUF header (maybe even 5, to be tested) */ + if (iSize+1 > srcSize) return ERROR(srcSize_wrong); + oSize = FSE_decompress_wksp(huffWeight, hwSize-1, ip+1, iSize, fseWorkspace, 6); /* max (hwSize-1) values decoded, as last one is implied */ + if (FSE_isError(oSize)) return oSize; + } + + /* collect weight stats */ + memset(rankStats, 0, (HUF_TABLELOG_MAX + 1) * sizeof(U32)); + weightTotal = 0; + { U32 n; for (n=0; n= HUF_TABLELOG_MAX) return ERROR(corruption_detected); + rankStats[huffWeight[n]]++; + weightTotal += (1 << huffWeight[n]) >> 1; + } } + if (weightTotal == 0) return ERROR(corruption_detected); + + /* get last non-null symbol weight (implied, total must be 2^n) */ + { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); + *tableLogPtr = tableLog; + /* determine last weight */ + { U32 const total = 1 << tableLog; + U32 const rest = total - weightTotal; + U32 const verif = 1 << BIT_highbit32(rest); + U32 const lastWeight = BIT_highbit32(rest) + 1; + if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ + huffWeight[oSize] = (BYTE)lastWeight; + rankStats[lastWeight]++; + } } + + /* check tree construction validity */ + if ((rankStats[1] < 2) || (rankStats[1] & 1)) return ERROR(corruption_detected); /* by construction : at least 2 elts of rank 1, must be even */ + + /* results */ + *nbSymbolsPtr = (U32)(oSize+1); + return iSize+1; +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/error_private.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/error_private.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/error_private.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/error_private.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,43 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* The purpose of this file is to have a single list of error strings embedded in binary */ + +#include "error_private.h" + +const char* ERR_getErrorString(ERR_enum code) +{ + static const char* const notErrorCode = "Unspecified error code"; + switch( code ) + { + case PREFIX(no_error): return "No error detected"; + case PREFIX(GENERIC): return "Error (generic)"; + case PREFIX(prefix_unknown): return "Unknown frame descriptor"; + case PREFIX(version_unsupported): return "Version not supported"; + case PREFIX(parameter_unknown): return "Unknown parameter type"; + case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; + case PREFIX(frameParameter_unsupportedBy32bits): return "Frame parameter unsupported in 32-bits mode"; + case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; + case PREFIX(compressionParameter_unsupported): return "Compression parameter is out of bound"; + case PREFIX(init_missing): return "Context should be init first"; + case PREFIX(memory_allocation): return "Allocation error : not enough memory"; + case PREFIX(stage_wrong): return "Operation not authorized at current processing stage"; + case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; + case PREFIX(srcSize_wrong): return "Src size incorrect"; + case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; + case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; + case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; + case PREFIX(dictionary_wrong): return "Dictionary mismatch"; + case PREFIX(maxCode): + default: return notErrorCode; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/error_private.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/error_private.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/error_private.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/error_private.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,76 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +/* Note : this module is expected to remain private, do not expose it */ + +#ifndef ERROR_H_MODULE +#define ERROR_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* **************************************** +* Dependencies +******************************************/ +#include /* size_t */ +#include "zstd_errors.h" /* enum list */ + + +/* **************************************** +* Compiler-specific +******************************************/ +#if defined(__GNUC__) +# define ERR_STATIC static __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define ERR_STATIC static inline +#elif defined(_MSC_VER) +# define ERR_STATIC static __inline +#else +# define ERR_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + + +/*-**************************************** +* Customization (error_public.h) +******************************************/ +typedef ZSTD_ErrorCode ERR_enum; +#define PREFIX(name) ZSTD_error_##name + + +/*-**************************************** +* Error codes handling +******************************************/ +#ifdef ERROR +# undef ERROR /* reported already defined on VS 2015 (Rich Geldreich) */ +#endif +#define ERROR(name) ((size_t)-PREFIX(name)) + +ERR_STATIC unsigned ERR_isError(size_t code) { return (code > ERROR(maxCode)); } + +ERR_STATIC ERR_enum ERR_getErrorCode(size_t code) { if (!ERR_isError(code)) return (ERR_enum)0; return (ERR_enum) (0-code); } + + +/*-**************************************** +* Error Strings +******************************************/ + +const char* ERR_getErrorString(ERR_enum code); /* error_private.c */ + +ERR_STATIC const char* ERR_getErrorName(size_t code) +{ + return ERR_getErrorString(ERR_getErrorCode(code)); +} + +#if defined (__cplusplus) +} +#endif + +#endif /* ERROR_H_MODULE */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/fse_decompress.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/fse_decompress.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/fse_decompress.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/fse_decompress.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,329 @@ +/* ****************************************************************** + FSE : Finite State Entropy decoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_isError ERR_isError +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + +/* check and forward error code */ +#define CHECK_F(f) { size_t const e = f; if (FSE_isError(e)) return e; } + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ +FSE_DTable* FSE_createDTable (unsigned tableLog) +{ + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + return (FSE_DTable*)malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); +} + +void FSE_freeDTable (FSE_DTable* dt) +{ + free(dt); +} + +size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ + FSE_DECODE_TYPE* const tableDecode = (FSE_DECODE_TYPE*) (tdPtr); + U16 symbolNext[FSE_MAX_SYMBOL_VALUE+1]; + + U32 const maxSV1 = maxSymbolValue + 1; + U32 const tableSize = 1 << tableLog; + U32 highThreshold = tableSize-1; + + /* Sanity Checks */ + if (maxSymbolValue > FSE_MAX_SYMBOL_VALUE) return ERROR(maxSymbolValue_tooLarge); + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + + /* Init, lay down lowprob symbols */ + { FSE_DTableHeader DTableH; + DTableH.tableLog = (U16)tableLog; + DTableH.fastMode = 1; + { S16 const largeLimit= (S16)(1 << (tableLog-1)); + U32 s; + for (s=0; s= largeLimit) DTableH.fastMode=0; + symbolNext[s] = normalizedCounter[s]; + } } } + memcpy(dt, &DTableH, sizeof(DTableH)); + } + + /* Spread symbols */ + { U32 const tableMask = tableSize-1; + U32 const step = FSE_TABLESTEP(tableSize); + U32 s, position = 0; + for (s=0; s highThreshold) position = (position + step) & tableMask; /* lowprob area */ + } } + if (position!=0) return ERROR(GENERIC); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ + } + + /* Build Decoding table */ + { U32 u; + for (u=0; utableLog = 0; + DTableH->fastMode = 0; + + cell->newState = 0; + cell->symbol = symbolValue; + cell->nbBits = 0; + + return 0; +} + + +size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) +{ + void* ptr = dt; + FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; + void* dPtr = dt + 1; + FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; + const unsigned tableSize = 1 << nbBits; + const unsigned tableMask = tableSize - 1; + const unsigned maxSV1 = tableMask+1; + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* Build Decoding Table */ + DTableH->tableLog = (U16)nbBits; + DTableH->fastMode = 1; + for (s=0; s sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[1] = FSE_GETSYMBOL(&state2); + + if (FSE_MAX_TABLELOG*4+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + { if (BIT_reloadDStream(&bitD) > BIT_DStream_unfinished) { op+=2; break; } } + + op[2] = FSE_GETSYMBOL(&state1); + + if (FSE_MAX_TABLELOG*2+7 > sizeof(bitD.bitContainer)*8) /* This test must be static */ + BIT_reloadDStream(&bitD); + + op[3] = FSE_GETSYMBOL(&state2); + } + + /* tail */ + /* note : BIT_reloadDStream(&bitD) >= FSE_DStream_partiallyFilled; Ends at exactly BIT_DStream_completed */ + while (1) { + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state1); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state2); + break; + } + + if (op>(omax-2)) return ERROR(dstSize_tooSmall); + *op++ = FSE_GETSYMBOL(&state2); + if (BIT_reloadDStream(&bitD)==BIT_DStream_overflow) { + *op++ = FSE_GETSYMBOL(&state1); + break; + } } + + return op-ostart; +} + + +size_t FSE_decompress_usingDTable(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize, + const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* DTableH = (const FSE_DTableHeader*)ptr; + const U32 fastMode = DTableH->fastMode; + + /* select fast mode (static) */ + if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); + return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); +} + + +size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, FSE_DTable* workSpace, unsigned maxLog) +{ + const BYTE* const istart = (const BYTE*)cSrc; + const BYTE* ip = istart; + short counting[FSE_MAX_SYMBOL_VALUE+1]; + unsigned tableLog; + unsigned maxSymbolValue = FSE_MAX_SYMBOL_VALUE; + + /* normal FSE decoding mode */ + size_t const NCountLength = FSE_readNCount (counting, &maxSymbolValue, &tableLog, istart, cSrcSize); + if (FSE_isError(NCountLength)) return NCountLength; + //if (NCountLength >= cSrcSize) return ERROR(srcSize_wrong); /* too small input size; supposed to be already checked in NCountLength, only remaining case : NCountLength==cSrcSize */ + if (tableLog > maxLog) return ERROR(tableLog_tooLarge); + ip += NCountLength; + cSrcSize -= NCountLength; + + CHECK_F( FSE_buildDTable (workSpace, counting, maxSymbolValue, tableLog) ); + + return FSE_decompress_usingDTable (dst, dstCapacity, ip, cSrcSize, workSpace); /* always return, even if it is an error code */ +} + + +typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; + +size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) +{ + DTable_max_t dt; /* Static analyzer seems unable to understand this table will be properly initialized later */ + return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, dt, FSE_MAX_TABLELOG); +} + + + +#endif /* FSE_COMMONDEFS_ONLY */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/fse.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/fse.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/fse.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/fse.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,668 @@ +/* ****************************************************************** + FSE : Finite State Entropy codec + Public Prototypes declaration + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef FSE_H +#define FSE_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*-***************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ + + +/*-**************************************** +* FSE simple functions +******************************************/ +/*! FSE_compress() : + Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. + 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). + @return : size of compressed data (<= dstCapacity). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. + if FSE_isError(return), compression failed (more details using FSE_getErrorName()) +*/ +size_t FSE_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/*! FSE_decompress(): + Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', + into already allocated destination buffer 'dst', of size 'dstCapacity'. + @return : size of regenerated data (<= maxDstSize), + or an error code, which can be tested using FSE_isError() . + + ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! + Why ? : making this distinction requires a header. + Header management is intentionally delegated to the user layer, which can better manage special cases. +*/ +size_t FSE_decompress(void* dst, size_t dstCapacity, + const void* cSrc, size_t cSrcSize); + + +/*-***************************************** +* Tool functions +******************************************/ +size_t FSE_compressBound(size_t size); /* maximum compressed size */ + +/* Error Management */ +unsigned FSE_isError(size_t code); /* tells if a return value is an error code */ +const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ + + +/*-***************************************** +* FSE advanced functions +******************************************/ +/*! FSE_compress2() : + Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' + Both parameters can be defined as '0' to mean : use default value + @return : size of compressed data + Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! + if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. + if FSE_isError(return), it's an error code. +*/ +size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + + +/*-***************************************** +* FSE detailed API +******************************************/ +/*! +FSE_compress() does the following: +1. count symbol occurrence from source[] into table count[] +2. normalize counters so that sum(count[]) == Power_of_2 (2^tableLog) +3. save normalized counters to memory buffer using writeNCount() +4. build encoding table 'CTable' from normalized counters +5. encode the data stream using encoding table 'CTable' + +FSE_decompress() does the following: +1. read normalized counters with readNCount() +2. build decoding table 'DTable' from normalized counters +3. decode the data stream using decoding table 'DTable' + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and provide normalized distribution using external method. +*/ + +/* *** COMPRESSION *** */ + +/*! FSE_count(): + Provides the precise count of each byte within a table 'count'. + 'count' is a table of unsigned int, of minimum size (*maxSymbolValuePtr+1). + *maxSymbolValuePtr will be updated if detected smaller than initial value. + @return : the count of the most frequent symbol (which is not identified). + if return == srcSize, there is only one symbol. + Can also return an error code, which can be tested with FSE_isError(). */ +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/*! FSE_optimalTableLog(): + dynamically downsize 'tableLog' when conditions are met. + It saves CPU time, by using smaller tables, while preserving or even improving compression ratio. + @return : recommended tableLog (necessarily <= 'maxTableLog') */ +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_normalizeCount(): + normalize counts so that sum(count[]) == Power_of_2 (2^tableLog) + 'normalizedCounter' is a table of short, of minimum size (maxSymbolValue+1). + @return : tableLog, + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_normalizeCount(short* normalizedCounter, unsigned tableLog, const unsigned* count, size_t srcSize, unsigned maxSymbolValue); + +/*! FSE_NCountWriteBound(): + Provides the maximum possible size of an FSE normalized table, given 'maxSymbolValue' and 'tableLog'. + Typically useful for allocation purpose. */ +size_t FSE_NCountWriteBound(unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_writeNCount(): + Compactly save 'normalizedCounter' into 'buffer'. + @return : size of the compressed table, + or an errorCode, which can be tested using FSE_isError(). */ +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + + +/*! Constructor and Destructor of FSE_CTable. + Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ +typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ +FSE_CTable* FSE_createCTable (unsigned tableLog, unsigned maxSymbolValue); +void FSE_freeCTable (FSE_CTable* ct); + +/*! FSE_buildCTable(): + Builds `ct`, which must be already allocated, using FSE_createCTable(). + @return : 0, or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_buildCTable(FSE_CTable* ct, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_compress_usingCTable(): + Compress `src` using `ct` into `dst` which must be already allocated. + @return : size of compressed data (<= `dstCapacity`), + or 0 if compressed data could not fit into `dst`, + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_compress_usingCTable (void* dst, size_t dstCapacity, const void* src, size_t srcSize, const FSE_CTable* ct); + +/*! +Tutorial : +---------- +The first step is to count all symbols. FSE_count() does this job very fast. +Result will be saved into 'count', a table of unsigned int, which must be already allocated, and have 'maxSymbolValuePtr[0]+1' cells. +'src' is a table of bytes of size 'srcSize'. All values within 'src' MUST be <= maxSymbolValuePtr[0] +maxSymbolValuePtr[0] will be updated, with its real value (necessarily <= original value) +FSE_count() will return the number of occurrence of the most frequent symbol. +This can be used to know if there is a single symbol within 'src', and to quickly evaluate its compressibility. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). + +The next step is to normalize the frequencies. +FSE_normalizeCount() will ensure that sum of frequencies is == 2 ^'tableLog'. +It also guarantees a minimum of 1 to any Symbol with frequency >= 1. +You can use 'tableLog'==0 to mean "use default tableLog value". +If you are unsure of which tableLog value to use, you can ask FSE_optimalTableLog(), +which will provide the optimal valid tableLog given sourceSize, maxSymbolValue, and a user-defined maximum (0 means "default"). + +The result of FSE_normalizeCount() will be saved into a table, +called 'normalizedCounter', which is a table of signed short. +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValue+1' cells. +The return value is tableLog if everything proceeded as expected. +It is 0 if there is a single symbol within distribution. +If there is an error (ex: invalid tableLog value), the function will return an ErrorCode (which can be tested using FSE_isError()). + +'normalizedCounter' can be saved in a compact manner to a memory area using FSE_writeNCount(). +'buffer' must be already allocated. +For guaranteed success, buffer size must be at least FSE_headerBound(). +The result of the function is the number of bytes written into 'buffer'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError(); ex : buffer size too small). + +'normalizedCounter' can then be used to create the compression table 'CTable'. +The space required by 'CTable' must be already allocated, using FSE_createCTable(). +You can then use FSE_buildCTable() to fill 'CTable'. +If there is an error, both functions will return an ErrorCode (which can be tested using FSE_isError()). + +'CTable' can then be used to compress 'src', with FSE_compress_usingCTable(). +Similar to FSE_count(), the convention is that 'src' is assumed to be a table of char of size 'srcSize' +The function returns the size of compressed data (without header), necessarily <= `dstCapacity`. +If it returns '0', compressed data could not fit into 'dst'. +If there is an error, the function will return an ErrorCode (which can be tested using FSE_isError()). +*/ + + +/* *** DECOMPRESSION *** */ + +/*! FSE_readNCount(): + Read compactly saved 'normalizedCounter' from 'rBuffer'. + @return : size read from 'rBuffer', + or an errorCode, which can be tested using FSE_isError(). + maxSymbolValuePtr[0] and tableLogPtr[0] will also be updated with their respective values */ +size_t FSE_readNCount (short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize); + +/*! Constructor and Destructor of FSE_DTable. + Note that its size depends on 'tableLog' */ +typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ +FSE_DTable* FSE_createDTable(unsigned tableLog); +void FSE_freeDTable(FSE_DTable* dt); + +/*! FSE_buildDTable(): + Builds 'dt', which must be already allocated, using FSE_createDTable(). + return : 0, or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); + +/*! FSE_decompress_usingDTable(): + Decompress compressed source `cSrc` of size `cSrcSize` using `dt` + into `dst` which must be already allocated. + @return : size of regenerated data (necessarily <= `dstCapacity`), + or an errorCode, which can be tested using FSE_isError() */ +size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); + +/*! +Tutorial : +---------- +(Note : these functions only decompress FSE-compressed blocks. + If block is uncompressed, use memcpy() instead + If block is a single repeated byte, use memset() instead ) + +The first step is to obtain the normalized frequencies of symbols. +This can be performed by FSE_readNCount() if it was saved using FSE_writeNCount(). +'normalizedCounter' must be already allocated, and have at least 'maxSymbolValuePtr[0]+1' cells of signed short. +In practice, that means it's necessary to know 'maxSymbolValue' beforehand, +or size the table to handle worst case situations (typically 256). +FSE_readNCount() will provide 'tableLog' and 'maxSymbolValue'. +The result of FSE_readNCount() is the number of bytes read from 'rBuffer'. +Note that 'rBufferSize' must be at least 4 bytes, even if useful information is less than that. +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +The next step is to build the decompression tables 'FSE_DTable' from 'normalizedCounter'. +This is performed by the function FSE_buildDTable(). +The space required by 'FSE_DTable' must be already allocated using FSE_createDTable(). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). + +`FSE_DTable` can then be used to decompress `cSrc`, with FSE_decompress_usingDTable(). +`cSrcSize` must be strictly correct, otherwise decompression will fail. +FSE_decompress_usingDTable() result will tell how many bytes were regenerated (<=`dstCapacity`). +If there is an error, the function will return an error code, which can be tested using FSE_isError(). (ex: dst buffer too small) +*/ + + +#ifdef FSE_STATIC_LINKING_ONLY + +/* *** Dependency *** */ +#include "bitstream.h" + + +/* ***************************************** +* Static allocation +*******************************************/ +/* FSE buffer bounds */ +#define FSE_NCOUNTBOUND 512 +#define FSE_BLOCKBOUND(size) (size + (size>>7)) +#define FSE_COMPRESSBOUND(size) (FSE_NCOUNTBOUND + FSE_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* It is possible to statically allocate FSE CTable/DTable as a table of FSE_CTable/FSE_DTable using below macros */ +#define FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) (1 + (1<<(maxTableLog-1)) + ((maxSymbolValue+1)*2)) +#define FSE_DTABLE_SIZE_U32(maxTableLog) (1 + (1<= `1024` unsigned + */ +size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace); + +/** FSE_countFast() : + * same as FSE_count(), but blindly trusts that all byte values within src are <= *maxSymbolValuePtr + */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + +/* FSE_countFast_wksp() : + * Same as FSE_countFast(), but using an externally provided scratch buffer. + * `workSpace` must be a table of minimum `1024` unsigned + */ +size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* workSpace); + +/*! FSE_count_simple + * Same as FSE_countFast(), but does not use any additional memory (not even on stack). + * This function is unsafe, and will segfault if any value within `src` is `> *maxSymbolValuePtr` (presuming it's also the size of `count`). +*/ +size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize); + + + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); +/**< same as FSE_optimalTableLog(), which used `minus==2` */ + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * FSE_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. + */ +#define FSE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + (1<<((maxTableLog>2)?(maxTableLog-2):0)) ) +size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); + +size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); +/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ + +size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); +/**< build a fake FSE_CTable, designed to compress always the same symbolValue */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` must be >= `(1<= BIT_DStream_completed + +When it's done, verify decompression is fully completed, by checking both DStream and the relevant states. +Checking if DStream has reached its end is performed by : + BIT_endOfDStream(&DStream); +Check also the states. There might be some symbols left there, if some high probability ones (>50%) are possible. + FSE_endOfDState(&DState); +*/ + + +/* ***************************************** +* FSE unsafe API +*******************************************/ +static unsigned char FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD); +/* faster, but works only if nbBits is always >= 1 (otherwise, result will be corrupted) */ + + +/* ***************************************** +* Implementation of inlined functions +*******************************************/ +typedef struct { + int deltaFindState; + U32 deltaNbBits; +} FSE_symbolCompressionTransform; /* total 8 bytes */ + +MEM_STATIC void FSE_initCState(FSE_CState_t* statePtr, const FSE_CTable* ct) +{ + const void* ptr = ct; + const U16* u16ptr = (const U16*) ptr; + const U32 tableLog = MEM_read16(ptr); + statePtr->value = (ptrdiff_t)1<stateTable = u16ptr+2; + statePtr->symbolTT = ((const U32*)ct + 1 + (tableLog ? (1<<(tableLog-1)) : 1)); + statePtr->stateLog = tableLog; +} + + +/*! FSE_initCState2() : +* Same as FSE_initCState(), but the first symbol to include (which will be the last to be read) +* uses the smallest state value possible, saving the cost of this symbol */ +MEM_STATIC void FSE_initCState2(FSE_CState_t* statePtr, const FSE_CTable* ct, U32 symbol) +{ + FSE_initCState(statePtr, ct); + { const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((symbolTT.deltaNbBits + (1<<15)) >> 16); + statePtr->value = (nbBitsOut << 16) - symbolTT.deltaNbBits; + statePtr->value = stateTable[(statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; + } +} + +MEM_STATIC void FSE_encodeSymbol(BIT_CStream_t* bitC, FSE_CState_t* statePtr, U32 symbol) +{ + const FSE_symbolCompressionTransform symbolTT = ((const FSE_symbolCompressionTransform*)(statePtr->symbolTT))[symbol]; + const U16* const stateTable = (const U16*)(statePtr->stateTable); + U32 nbBitsOut = (U32)((statePtr->value + symbolTT.deltaNbBits) >> 16); + BIT_addBits(bitC, statePtr->value, nbBitsOut); + statePtr->value = stateTable[ (statePtr->value >> nbBitsOut) + symbolTT.deltaFindState]; +} + +MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePtr) +{ + BIT_addBits(bitC, statePtr->value, statePtr->stateLog); + BIT_flushBits(bitC); +} + + +/* ====== Decompression ====== */ + +typedef struct { + U16 tableLog; + U16 fastMode; +} FSE_DTableHeader; /* sizeof U32 */ + +typedef struct +{ + unsigned short newState; + unsigned char symbol; + unsigned char nbBits; +} FSE_decode_t; /* size == U32 */ + +MEM_STATIC void FSE_initDState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD, const FSE_DTable* dt) +{ + const void* ptr = dt; + const FSE_DTableHeader* const DTableH = (const FSE_DTableHeader*)ptr; + DStatePtr->state = BIT_readBits(bitD, DTableH->tableLog); + BIT_reloadDStream(bitD); + DStatePtr->table = dt + 1; +} + +MEM_STATIC BYTE FSE_peekSymbol(const FSE_DState_t* DStatePtr) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + return DInfo.symbol; +} + +MEM_STATIC void FSE_updateState(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + size_t const lowBits = BIT_readBits(bitD, nbBits); + DStatePtr->state = DInfo.newState + lowBits; +} + +MEM_STATIC BYTE FSE_decodeSymbol(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBits(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +/*! FSE_decodeSymbolFast() : + unsafe, only works if no symbol has a probability > 50% */ +MEM_STATIC BYTE FSE_decodeSymbolFast(FSE_DState_t* DStatePtr, BIT_DStream_t* bitD) +{ + FSE_decode_t const DInfo = ((const FSE_decode_t*)(DStatePtr->table))[DStatePtr->state]; + U32 const nbBits = DInfo.nbBits; + BYTE const symbol = DInfo.symbol; + size_t const lowBits = BIT_readBitsFast(bitD, nbBits); + + DStatePtr->state = DInfo.newState + lowBits; + return symbol; +} + +MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) +{ + return DStatePtr->state == 0; +} + + + +#ifndef FSE_COMMONDEFS_ONLY + +/* ************************************************************** +* Tuning parameters +****************************************************************/ +/*!MEMORY_USAGE : +* Memory usage formula : N->2^N Bytes (examples : 10 -> 1KB; 12 -> 4KB ; 16 -> 64KB; 20 -> 1MB; etc.) +* Increasing memory usage improves compression ratio +* Reduced memory usage can improve speed, due to cache effect +* Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ +#ifndef FSE_MAX_MEMORY_USAGE +# define FSE_MAX_MEMORY_USAGE 14 +#endif +#ifndef FSE_DEFAULT_MEMORY_USAGE +# define FSE_DEFAULT_MEMORY_USAGE 13 +#endif + +/*!FSE_MAX_SYMBOL_VALUE : +* Maximum symbol value authorized. +* Required for proper stack allocation */ +#ifndef FSE_MAX_SYMBOL_VALUE +# define FSE_MAX_SYMBOL_VALUE 255 +#endif + +/* ************************************************************** +* template functions type & suffix +****************************************************************/ +#define FSE_FUNCTION_TYPE BYTE +#define FSE_FUNCTION_EXTENSION +#define FSE_DECODE_TYPE FSE_decode_t + + +#endif /* !FSE_COMMONDEFS_ONLY */ + + +/* *************************************************************** +* Constants +*****************************************************************/ +#define FSE_MAX_TABLELOG (FSE_MAX_MEMORY_USAGE-2) +#define FSE_MAX_TABLESIZE (1U< FSE_TABLELOG_ABSOLUTE_MAX +# error "FSE_MAX_TABLELOG > FSE_TABLELOG_ABSOLUTE_MAX is not supported" +#endif + +#define FSE_TABLESTEP(tableSize) ((tableSize>>1) + (tableSize>>3) + 3) + + +#endif /* FSE_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* FSE_H */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/huf.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/huf.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/huf.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/huf.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,238 @@ +/* ****************************************************************** + Huffman coder, part of New Generation Entropy library + header file + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - Source repository : https://github.com/Cyan4973/FiniteStateEntropy +****************************************************************** */ +#ifndef HUF_H_298734234 +#define HUF_H_298734234 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/* *** Dependencies *** */ +#include /* size_t */ + + +/* *** simple functions *** */ +/** +HUF_compress() : + Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. + 'dst' buffer must be already allocated. + Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). + `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. + @return : size of compressed data (<= `dstCapacity`). + Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! + if return == 1, srcData is a single repeated byte symbol (RLE compression). + if HUF_isError(return), compression failed (more details using HUF_getErrorName()) +*/ +size_t HUF_compress(void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +/** +HUF_decompress() : + Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', + into already allocated buffer 'dst', of minimum size 'dstSize'. + `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. + Note : in contrast with FSE, HUF_decompress can regenerate + RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, + because it knows size to regenerate. + @return : size of regenerated data (== originalSize), + or an error code, which can be tested using HUF_isError() +*/ +size_t HUF_decompress(void* dst, size_t originalSize, + const void* cSrc, size_t cSrcSize); + + +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +/* *** Advanced function *** */ + +/** HUF_compress2() : + * Same as HUF_compress(), but offers direct control over `maxSymbolValue` and `tableLog` . + * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ +size_t HUF_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); + +/** HUF_compress4X_wksp() : +* Same as HUF_compress2(), but uses externally allocated `workSpace`, which must be a table of >= 1024 unsigned */ +size_t HUF_compress4X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */ + + + +#ifdef HUF_STATIC_LINKING_ONLY + +/* *** Dependencies *** */ +#include "mem.h" /* U32 */ + + +/* *** Constants *** */ +#define HUF_TABLELOG_ABSOLUTEMAX 15 /* absolute limit of HUF_MAX_TABLELOG. Beyond that value, code does not work */ +#define HUF_TABLELOG_MAX 12 /* max configured tableLog (for static allocation); can be modified up to HUF_ABSOLUTEMAX_TABLELOG */ +#define HUF_TABLELOG_DEFAULT 11 /* tableLog by default, when not specified */ +#define HUF_SYMBOLVALUE_MAX 255 +#if (HUF_TABLELOG_MAX > HUF_TABLELOG_ABSOLUTEMAX) +# error "HUF_TABLELOG_MAX is too large !" +#endif + + +/* **************************************** +* Static allocation +******************************************/ +/* HUF buffer bounds */ +#define HUF_CTABLEBOUND 129 +#define HUF_BLOCKBOUND(size) (size + (size>>8) + 8) /* only true if incompressible pre-filtered with fast heuristic */ +#define HUF_COMPRESSBOUND(size) (HUF_CTABLEBOUND + HUF_BLOCKBOUND(size)) /* Macro version, useful for static allocation */ + +/* static allocation of HUF's Compression Table */ +#define HUF_CREATE_STATIC_CTABLE(name, maxSymbolValue) \ + U32 name##hb[maxSymbolValue+1]; \ + void* name##hv = &(name##hb); \ + HUF_CElt* name = (HUF_CElt*)(name##hv) /* no final ; */ + +/* static allocation of HUF's DTable */ +typedef U32 HUF_DTable; +#define HUF_DTABLE_SIZE(maxTableLog) (1 + (1<<(maxTableLog))) +#define HUF_CREATE_STATIC_DTABLEX2(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE((maxTableLog)-1)] = { ((U32)((maxTableLog)-1) * 0x01000001) } +#define HUF_CREATE_STATIC_DTABLEX4(DTable, maxTableLog) \ + HUF_DTable DTable[HUF_DTABLE_SIZE(maxTableLog)] = { ((U32)(maxTableLog) * 0x01000001) } + + +/* **************************************** +* Advanced decompression functions +******************************************/ +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ +size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ +size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress4X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + + +/* **************************************** +* HUF detailed API +******************************************/ +/*! +HUF_compress() does the following: +1. count symbol occurrence from source[] into table count[] using FSE_count() +2. (optional) refine tableLog using HUF_optimalTableLog() +3. build Huffman table from count using HUF_buildCTable() +4. save Huffman table to memory buffer using HUF_writeCTable() +5. encode the data stream using HUF_compress4X_usingCTable() + +The following API allows targeting specific sub-functions for advanced tasks. +For example, it's possible to compress several blocks using the same 'CTable', +or to save and regenerate 'CTable' using external methods. +*/ +/* FSE_count() : find it within "fse.h" */ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); +typedef struct HUF_CElt_s HUF_CElt; /* incomplete type */ +size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); +size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. + */ +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize); + +/*! HUF_readStats() : + Read compact Huffman tree, saved by HUF_writeCTable(). + `huffWeight` is destination buffer. + @return : size read from `src` , or an error Code . + Note : Needed by HUF_readCTable() and HUF_readDTableXn() . */ +size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, + U32* nbSymbolsPtr, U32* tableLogPtr, + const void* src, size_t srcSize); + +/** HUF_readCTable() : +* Loading a CTable saved with HUF_writeCTable() */ +size_t HUF_readCTable (HUF_CElt* CTable, unsigned maxSymbolValue, const void* src, size_t srcSize); + + +/* +HUF_decompress() does the following: +1. select the decompression algorithm (X2, X4) based on pre-computed heuristics +2. build Huffman table from save, using HUF_readDTableXn() +3. decode 1 or 4 segments in parallel using HUF_decompressSXn_usingDTable +*/ + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize); + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + + +/* single stream variants */ + +size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least 1024 unsigned */ +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); +size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ +size_t HUF_decompress1X4_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ +size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); + +#endif /* HUF_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* HUF_H_298734234 */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/mem.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/mem.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/mem.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/mem.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,372 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef MEM_H_MODULE +#define MEM_H_MODULE + +#if defined (__cplusplus) +extern "C" { +#endif + +/*-**************************************** +* Dependencies +******************************************/ +#include /* size_t, ptrdiff_t */ +#include /* memcpy */ + + +/*-**************************************** +* Compiler specifics +******************************************/ +#if defined(_MSC_VER) /* Visual Studio */ +# include /* _byteswap_ulong */ +# include /* _byteswap_* */ +#endif +#if defined(__GNUC__) +# define MEM_STATIC static __inline __attribute__((unused)) +#elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define MEM_STATIC static inline +#elif defined(_MSC_VER) +# define MEM_STATIC static __inline +#else +# define MEM_STATIC static /* this version may generate warnings for unused static functions; disable the relevant warning */ +#endif + +/* code only tested on 32 and 64 bits systems */ +#define MEM_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } +MEM_STATIC void MEM_check(void) { MEM_STATIC_ASSERT((sizeof(size_t)==4) || (sizeof(size_t)==8)); } + + +/*-************************************************************** +* Basic Types +*****************************************************************/ +#if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef int16_t S16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; + typedef int64_t S64; + typedef intptr_t iPtrDiff; +#else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef signed short S16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; + typedef signed long long S64; + typedef ptrdiff_t iPtrDiff; +#endif + + +/*-************************************************************** +* Memory I/O +*****************************************************************/ +/* MEM_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method is portable but violate C standard. + * It can generate buggy code on targets depending on alignment. + * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define MEM_FORCE_MEMORY_ACCESS 2 +# elif defined(__INTEL_COMPILER) /*|| defined(_MSC_VER)*/ || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define MEM_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } +MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } + +MEM_STATIC unsigned MEM_isLittleEndian(void) +{ + const union { U32 u; BYTE c[4]; } one = { 1 }; /* don't use static : performance detrimental */ + return one.c[0]; +} + +#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) + +/* violates C standard, by lying on structure alignment. +Only use if no other choice to achieve best performance on target platform */ +MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } +MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } +MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } +MEM_STATIC U64 MEM_readST(const void* memPtr) { return *(const size_t*) memPtr; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } + +#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) + __pragma( pack(push, 1) ) + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } unalign; + __pragma( pack(pop) ) +#else + typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; +#endif + +MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } +MEM_STATIC U64 MEM_readST(const void* ptr) { return ((const unalign*)ptr)->st; } + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } + +#else + +/* default method, safe and standard. + can sometimes prove slower */ + +MEM_STATIC U16 MEM_read16(const void* memPtr) +{ + U16 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U32 MEM_read32(const void* memPtr) +{ + U32 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC U64 MEM_read64(const void* memPtr) +{ + U64 val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC size_t MEM_readST(const void* memPtr) +{ + size_t val; memcpy(&val, memPtr, sizeof(val)); return val; +} + +MEM_STATIC void MEM_write16(void* memPtr, U16 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write32(void* memPtr, U32 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +MEM_STATIC void MEM_write64(void* memPtr, U64 value) +{ + memcpy(memPtr, &value, sizeof(value)); +} + +#endif /* MEM_FORCE_MEMORY_ACCESS */ + +MEM_STATIC U32 MEM_swap32(U32 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_ulong(in); +#elif defined (__GNUC__) + return __builtin_bswap32(in); +#else + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +#endif +} + +MEM_STATIC U64 MEM_swap64(U64 in) +{ +#if defined(_MSC_VER) /* Visual Studio */ + return _byteswap_uint64(in); +#elif defined (__GNUC__) + return __builtin_bswap64(in); +#else + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +#endif +} + +MEM_STATIC size_t MEM_swapST(size_t in) +{ + if (MEM_32bits()) + return (size_t)MEM_swap32((U32)in); + else + return (size_t)MEM_swap64((U64)in); +} + +/*=== Little endian r/w ===*/ + +MEM_STATIC U16 MEM_readLE16(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read16(memPtr); + else { + const BYTE* p = (const BYTE*)memPtr; + return (U16)(p[0] + (p[1]<<8)); + } +} + +MEM_STATIC void MEM_writeLE16(void* memPtr, U16 val) +{ + if (MEM_isLittleEndian()) { + MEM_write16(memPtr, val); + } else { + BYTE* p = (BYTE*)memPtr; + p[0] = (BYTE)val; + p[1] = (BYTE)(val>>8); + } +} + +MEM_STATIC U32 MEM_readLE24(const void* memPtr) +{ + return MEM_readLE16(memPtr) + (((const BYTE*)memPtr)[2] << 16); +} + +MEM_STATIC void MEM_writeLE24(void* memPtr, U32 val) +{ + MEM_writeLE16(memPtr, (U16)val); + ((BYTE*)memPtr)[2] = (BYTE)(val>>16); +} + +MEM_STATIC U32 MEM_readLE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read32(memPtr); + else + return MEM_swap32(MEM_read32(memPtr)); +} + +MEM_STATIC void MEM_writeLE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, val32); + else + MEM_write32(memPtr, MEM_swap32(val32)); +} + +MEM_STATIC U64 MEM_readLE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_read64(memPtr); + else + return MEM_swap64(MEM_read64(memPtr)); +} + +MEM_STATIC void MEM_writeLE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, val64); + else + MEM_write64(memPtr, MEM_swap64(val64)); +} + +MEM_STATIC size_t MEM_readLEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readLE32(memPtr); + else + return (size_t)MEM_readLE64(memPtr); +} + +MEM_STATIC void MEM_writeLEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeLE32(memPtr, (U32)val); + else + MEM_writeLE64(memPtr, (U64)val); +} + +/*=== Big endian r/w ===*/ + +MEM_STATIC U32 MEM_readBE32(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap32(MEM_read32(memPtr)); + else + return MEM_read32(memPtr); +} + +MEM_STATIC void MEM_writeBE32(void* memPtr, U32 val32) +{ + if (MEM_isLittleEndian()) + MEM_write32(memPtr, MEM_swap32(val32)); + else + MEM_write32(memPtr, val32); +} + +MEM_STATIC U64 MEM_readBE64(const void* memPtr) +{ + if (MEM_isLittleEndian()) + return MEM_swap64(MEM_read64(memPtr)); + else + return MEM_read64(memPtr); +} + +MEM_STATIC void MEM_writeBE64(void* memPtr, U64 val64) +{ + if (MEM_isLittleEndian()) + MEM_write64(memPtr, MEM_swap64(val64)); + else + MEM_write64(memPtr, val64); +} + +MEM_STATIC size_t MEM_readBEST(const void* memPtr) +{ + if (MEM_32bits()) + return (size_t)MEM_readBE32(memPtr); + else + return (size_t)MEM_readBE64(memPtr); +} + +MEM_STATIC void MEM_writeBEST(void* memPtr, size_t val) +{ + if (MEM_32bits()) + MEM_writeBE32(memPtr, (U32)val); + else + MEM_writeBE64(memPtr, (U64)val); +} + + +/* function safe only for comparisons */ +MEM_STATIC U32 MEM_readMINMATCH(const void* memPtr, U32 length) +{ + switch (length) + { + default : + case 4 : return MEM_read32(memPtr); + case 3 : if (MEM_isLittleEndian()) + return MEM_read32(memPtr)<<8; + else + return MEM_read32(memPtr)>>8; + } +} + +#if defined (__cplusplus) +} +#endif + +#endif /* MEM_H_MODULE */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/xxhash.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/xxhash.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/xxhash.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/xxhash.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,867 @@ +/* +* xxHash - Fast Hash algorithm +* Copyright (C) 2012-2016, Yann Collet +* +* BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) +* +* Redistribution and use in source and binary forms, with or without +* modification, are permitted provided that the following conditions are +* met: +* +* * Redistributions of source code must retain the above copyright +* notice, this list of conditions and the following disclaimer. +* * Redistributions in binary form must reproduce the above +* copyright notice, this list of conditions and the following disclaimer +* in the documentation and/or other materials provided with the +* distribution. +* +* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +* +* You can contact the author at : +* - xxHash homepage: http://www.xxhash.com +* - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + + +/* ************************************* +* Tuning parameters +***************************************/ +/*!XXH_FORCE_MEMORY_ACCESS : + * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. + * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. + * The below switch allow to select different access method for improved performance. + * Method 0 (default) : use `memcpy()`. Safe and portable. + * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). + * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. + * Method 2 : direct access. This method doesn't depend on compiler but violate C standard. + * It can generate buggy code on targets which do not support unaligned memory accesses. + * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) + * See http://stackoverflow.com/a/32095106/646947 for details. + * Prefer these methods in priority order (0 > 1 > 2) + */ +#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ +# if defined(__GNUC__) && ( defined(__ARM_ARCH_6__) || defined(__ARM_ARCH_6J__) || defined(__ARM_ARCH_6K__) || defined(__ARM_ARCH_6Z__) || defined(__ARM_ARCH_6ZK__) || defined(__ARM_ARCH_6T2__) ) +# define XXH_FORCE_MEMORY_ACCESS 2 +# elif (defined(__INTEL_COMPILER) && !defined(WIN32)) || \ + (defined(__GNUC__) && ( defined(__ARM_ARCH_7__) || defined(__ARM_ARCH_7A__) || defined(__ARM_ARCH_7R__) || defined(__ARM_ARCH_7M__) || defined(__ARM_ARCH_7S__) )) +# define XXH_FORCE_MEMORY_ACCESS 1 +# endif +#endif + +/*!XXH_ACCEPT_NULL_INPUT_POINTER : + * If the input pointer is a null pointer, xxHash default behavior is to trigger a memory access error, since it is a bad pointer. + * When this option is enabled, xxHash output for null input pointers will be the same as a null-length input. + * By default, this option is disabled. To enable it, uncomment below define : + */ +/* #define XXH_ACCEPT_NULL_INPUT_POINTER 1 */ + +/*!XXH_FORCE_NATIVE_FORMAT : + * By default, xxHash library provides endian-independant Hash values, based on little-endian convention. + * Results are therefore identical for little-endian and big-endian CPU. + * This comes at a performance cost for big-endian CPU, since some swapping is required to emulate little-endian format. + * Should endian-independance be of no importance for your application, you may set the #define below to 1, + * to improve speed for Big-endian CPU. + * This option has no impact on Little_Endian CPU. + */ +#ifndef XXH_FORCE_NATIVE_FORMAT /* can be defined externally */ +# define XXH_FORCE_NATIVE_FORMAT 0 +#endif + +/*!XXH_FORCE_ALIGN_CHECK : + * This is a minor performance trick, only useful with lots of very small keys. + * It means : check for aligned/unaligned input. + * The check costs one initial branch per hash; set to 0 when the input data + * is guaranteed to be aligned. + */ +#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */ +# if defined(__i386) || defined(_M_IX86) || defined(__x86_64__) || defined(_M_X64) +# define XXH_FORCE_ALIGN_CHECK 0 +# else +# define XXH_FORCE_ALIGN_CHECK 1 +# endif +#endif + + +/* ************************************* +* Includes & Memory related functions +***************************************/ +/* Modify the local functions below should you wish to use some other memory routines */ +/* for malloc(), free() */ +#include +static void* XXH_malloc(size_t s) { return malloc(s); } +static void XXH_free (void* p) { free(p); } +/* for memcpy() */ +#include +static void* XXH_memcpy(void* dest, const void* src, size_t size) { return memcpy(dest,src,size); } + +#define XXH_STATIC_LINKING_ONLY +#include "xxhash.h" + + +/* ************************************* +* Compiler Specific Options +***************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# define FORCE_INLINE static __forceinline +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************* +* Basic Types +***************************************/ +#ifndef MEM_MODULE +# define MEM_MODULE +# if !defined (__VMS) && (defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) ) +# include + typedef uint8_t BYTE; + typedef uint16_t U16; + typedef uint32_t U32; + typedef int32_t S32; + typedef uint64_t U64; +# else + typedef unsigned char BYTE; + typedef unsigned short U16; + typedef unsigned int U32; + typedef signed int S32; + typedef unsigned long long U64; /* if your compiler doesn't support unsigned long long, replace by another 64-bit type here. Note that xxhash.h will also need to be updated. */ +# endif +#endif + + +#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2)) + +/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */ +static U32 XXH_read32(const void* memPtr) { return *(const U32*) memPtr; } +static U64 XXH_read64(const void* memPtr) { return *(const U64*) memPtr; } + +#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1)) + +/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ +/* currently only defined for gcc and icc */ +typedef union { U32 u32; U64 u64; } __attribute__((packed)) unalign; + +static U32 XXH_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } +static U64 XXH_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } + +#else + +/* portable and safe solution. Generally efficient. + * see : http://stackoverflow.com/a/32095106/646947 + */ + +static U32 XXH_read32(const void* memPtr) +{ + U32 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +static U64 XXH_read64(const void* memPtr) +{ + U64 val; + memcpy(&val, memPtr, sizeof(val)); + return val; +} + +#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */ + + +/* **************************************** +* Compiler-specific Functions and Macros +******************************************/ +#define GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) + +/* Note : although _rotl exists for minGW (GCC under windows), performance seems poor */ +#if defined(_MSC_VER) +# define XXH_rotl32(x,r) _rotl(x,r) +# define XXH_rotl64(x,r) _rotl64(x,r) +#else +# define XXH_rotl32(x,r) ((x << r) | (x >> (32 - r))) +# define XXH_rotl64(x,r) ((x << r) | (x >> (64 - r))) +#endif + +#if defined(_MSC_VER) /* Visual Studio */ +# define XXH_swap32 _byteswap_ulong +# define XXH_swap64 _byteswap_uint64 +#elif GCC_VERSION >= 403 +# define XXH_swap32 __builtin_bswap32 +# define XXH_swap64 __builtin_bswap64 +#else +static U32 XXH_swap32 (U32 x) +{ + return ((x << 24) & 0xff000000 ) | + ((x << 8) & 0x00ff0000 ) | + ((x >> 8) & 0x0000ff00 ) | + ((x >> 24) & 0x000000ff ); +} +static U64 XXH_swap64 (U64 x) +{ + return ((x << 56) & 0xff00000000000000ULL) | + ((x << 40) & 0x00ff000000000000ULL) | + ((x << 24) & 0x0000ff0000000000ULL) | + ((x << 8) & 0x000000ff00000000ULL) | + ((x >> 8) & 0x00000000ff000000ULL) | + ((x >> 24) & 0x0000000000ff0000ULL) | + ((x >> 40) & 0x000000000000ff00ULL) | + ((x >> 56) & 0x00000000000000ffULL); +} +#endif + + +/* ************************************* +* Architecture Macros +***************************************/ +typedef enum { XXH_bigEndian=0, XXH_littleEndian=1 } XXH_endianess; + +/* XXH_CPU_LITTLE_ENDIAN can be defined externally, for example on the compiler command line */ +#ifndef XXH_CPU_LITTLE_ENDIAN + static const int g_one = 1; +# define XXH_CPU_LITTLE_ENDIAN (*(const char*)(&g_one)) +#endif + + +/* *************************** +* Memory reads +*****************************/ +typedef enum { XXH_aligned, XXH_unaligned } XXH_alignment; + +FORCE_INLINE U32 XXH_readLE32_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr)); + else + return endian==XXH_littleEndian ? *(const U32*)ptr : XXH_swap32(*(const U32*)ptr); +} + +FORCE_INLINE U32 XXH_readLE32(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE32_align(ptr, endian, XXH_unaligned); +} + +static U32 XXH_readBE32(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr); +} + +FORCE_INLINE U64 XXH_readLE64_align(const void* ptr, XXH_endianess endian, XXH_alignment align) +{ + if (align==XXH_unaligned) + return endian==XXH_littleEndian ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr)); + else + return endian==XXH_littleEndian ? *(const U64*)ptr : XXH_swap64(*(const U64*)ptr); +} + +FORCE_INLINE U64 XXH_readLE64(const void* ptr, XXH_endianess endian) +{ + return XXH_readLE64_align(ptr, endian, XXH_unaligned); +} + +static U64 XXH_readBE64(const void* ptr) +{ + return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr); +} + + +/* ************************************* +* Macros +***************************************/ +#define XXH_STATIC_ASSERT(c) { enum { XXH_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************* +* Constants +***************************************/ +static const U32 PRIME32_1 = 2654435761U; +static const U32 PRIME32_2 = 2246822519U; +static const U32 PRIME32_3 = 3266489917U; +static const U32 PRIME32_4 = 668265263U; +static const U32 PRIME32_5 = 374761393U; + +static const U64 PRIME64_1 = 11400714785074694791ULL; +static const U64 PRIME64_2 = 14029467366897019727ULL; +static const U64 PRIME64_3 = 1609587929392839161ULL; +static const U64 PRIME64_4 = 9650029242287828579ULL; +static const U64 PRIME64_5 = 2870177450012600261ULL; + +XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; } + + +/* ************************** +* Utils +****************************/ +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dstState, const XXH32_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dstState, const XXH64_state_t* restrict srcState) +{ + memcpy(dstState, srcState, sizeof(*dstState)); +} + + +/* *************************** +* Simple Hash Functions +*****************************/ + +static U32 XXH32_round(U32 seed, U32 input) +{ + seed += input * PRIME32_2; + seed = XXH_rotl32(seed, 13); + seed *= PRIME32_1; + return seed; +} + +FORCE_INLINE U32 XXH32_endian_align(const void* input, size_t len, U32 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* bEnd = p + len; + U32 h32; +#define XXH_get32bits(p) XXH_readLE32_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)16; + } +#endif + + if (len>=16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = seed + PRIME32_1 + PRIME32_2; + U32 v2 = seed + PRIME32_2; + U32 v3 = seed + 0; + U32 v4 = seed - PRIME32_1; + + do { + v1 = XXH32_round(v1, XXH_get32bits(p)); p+=4; + v2 = XXH32_round(v2, XXH_get32bits(p)); p+=4; + v3 = XXH32_round(v3, XXH_get32bits(p)); p+=4; + v4 = XXH32_round(v4, XXH_get32bits(p)); p+=4; + } while (p<=limit); + + h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7) + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18); + } else { + h32 = seed + PRIME32_5; + } + + h32 += (U32) len; + + while (p+4<=bEnd) { + h32 += XXH_get32bits(p) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4 ; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32 (const void* input, size_t len, unsigned int seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH32_CREATESTATE_STATIC(state); + XXH32_reset(state, seed); + XXH32_update(state, input, len); + return XXH32_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH32_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +static U64 XXH64_round(U64 acc, U64 input) +{ + acc += input * PRIME64_2; + acc = XXH_rotl64(acc, 31); + acc *= PRIME64_1; + return acc; +} + +static U64 XXH64_mergeRound(U64 acc, U64 val) +{ + val = XXH64_round(0, val); + acc ^= val; + acc = acc * PRIME64_1 + PRIME64_4; + return acc; +} + +FORCE_INLINE U64 XXH64_endian_align(const void* input, size_t len, U64 seed, XXH_endianess endian, XXH_alignment align) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + U64 h64; +#define XXH_get64bits(p) XXH_readLE64_align(p, endian, align) + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (p==NULL) { + len=0; + bEnd=p=(const BYTE*)(size_t)32; + } +#endif + + if (len>=32) { + const BYTE* const limit = bEnd - 32; + U64 v1 = seed + PRIME64_1 + PRIME64_2; + U64 v2 = seed + PRIME64_2; + U64 v3 = seed + 0; + U64 v4 = seed - PRIME64_1; + + do { + v1 = XXH64_round(v1, XXH_get64bits(p)); p+=8; + v2 = XXH64_round(v2, XXH_get64bits(p)); p+=8; + v3 = XXH64_round(v3, XXH_get64bits(p)); p+=8; + v4 = XXH64_round(v4, XXH_get64bits(p)); p+=8; + } while (p<=limit); + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + + } else { + h64 = seed + PRIME64_5; + } + + h64 += (U64) len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_get64bits(p)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_get32bits(p)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64 (const void* input, size_t len, unsigned long long seed) +{ +#if 0 + /* Simple version, good for code maintenance, but unfortunately slow for small inputs */ + XXH64_CREATESTATE_STATIC(state); + XXH64_reset(state, seed); + XXH64_update(state, input, len); + return XXH64_digest(state); +#else + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if (XXH_FORCE_ALIGN_CHECK) { + if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */ + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_aligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_aligned); + } } + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_endian_align(input, len, seed, XXH_littleEndian, XXH_unaligned); + else + return XXH64_endian_align(input, len, seed, XXH_bigEndian, XXH_unaligned); +#endif +} + + +/* ************************************************** +* Advanced Hash Functions +****************************************************/ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void) +{ + return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void) +{ + return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t)); +} +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr) +{ + XXH_free(statePtr); + return XXH_OK; +} + + +/*** Hash feed ***/ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, unsigned int seed) +{ + XXH32_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-4); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME32_1 + PRIME32_2; + state.v2 = seed + PRIME32_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME32_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH64_state_t* statePtr, unsigned long long seed) +{ + XXH64_state_t state; /* using a local state to memcpy() in order to avoid strict-aliasing warnings */ + memset(&state, 0, sizeof(state)-8); /* do not write into reserved, for future removal */ + state.v1 = seed + PRIME64_1 + PRIME64_2; + state.v2 = seed + PRIME64_2; + state.v3 = seed + 0; + state.v4 = seed - PRIME64_1; + memcpy(statePtr, &state, sizeof(state)); + return XXH_OK; +} + + +FORCE_INLINE XXH_errorcode XXH32_update_endian (XXH32_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len_32 += (unsigned)len; + state->large_len |= (len>=16) | (state->total_len_32>=16); + + if (state->memsize + len < 16) { /* fill in tmp buffer */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, len); + state->memsize += (unsigned)len; + return XXH_OK; + } + + if (state->memsize) { /* some data left from previous update */ + XXH_memcpy((BYTE*)(state->mem32) + state->memsize, input, 16-state->memsize); + { const U32* p32 = state->mem32; + state->v1 = XXH32_round(state->v1, XXH_readLE32(p32, endian)); p32++; + state->v2 = XXH32_round(state->v2, XXH_readLE32(p32, endian)); p32++; + state->v3 = XXH32_round(state->v3, XXH_readLE32(p32, endian)); p32++; + state->v4 = XXH32_round(state->v4, XXH_readLE32(p32, endian)); p32++; + } + p += 16-state->memsize; + state->memsize = 0; + } + + if (p <= bEnd-16) { + const BYTE* const limit = bEnd - 16; + U32 v1 = state->v1; + U32 v2 = state->v2; + U32 v3 = state->v3; + U32 v4 = state->v4; + + do { + v1 = XXH32_round(v1, XXH_readLE32(p, endian)); p+=4; + v2 = XXH32_round(v2, XXH_readLE32(p, endian)); p+=4; + v3 = XXH32_round(v3, XXH_readLE32(p, endian)); p+=4; + v4 = XXH32_round(v4, XXH_readLE32(p, endian)); p+=4; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem32, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH32_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U32 XXH32_digest_endian (const XXH32_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem32; + const BYTE* const bEnd = (const BYTE*)(state->mem32) + state->memsize; + U32 h32; + + if (state->large_len) { + h32 = XXH_rotl32(state->v1, 1) + XXH_rotl32(state->v2, 7) + XXH_rotl32(state->v3, 12) + XXH_rotl32(state->v4, 18); + } else { + h32 = state->v3 /* == seed */ + PRIME32_5; + } + + h32 += state->total_len_32; + + while (p+4<=bEnd) { + h32 += XXH_readLE32(p, endian) * PRIME32_3; + h32 = XXH_rotl32(h32, 17) * PRIME32_4; + p+=4; + } + + while (p> 15; + h32 *= PRIME32_2; + h32 ^= h32 >> 13; + h32 *= PRIME32_3; + h32 ^= h32 >> 16; + + return h32; +} + + +XXH_PUBLIC_API unsigned int XXH32_digest (const XXH32_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH32_digest_endian(state_in, XXH_littleEndian); + else + return XXH32_digest_endian(state_in, XXH_bigEndian); +} + + + +/* **** XXH64 **** */ + +FORCE_INLINE XXH_errorcode XXH64_update_endian (XXH64_state_t* state, const void* input, size_t len, XXH_endianess endian) +{ + const BYTE* p = (const BYTE*)input; + const BYTE* const bEnd = p + len; + +#ifdef XXH_ACCEPT_NULL_INPUT_POINTER + if (input==NULL) return XXH_ERROR; +#endif + + state->total_len += len; + + if (state->memsize + len < 32) { /* fill in tmp buffer */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, len); + state->memsize += (U32)len; + return XXH_OK; + } + + if (state->memsize) { /* tmp buffer is full */ + XXH_memcpy(((BYTE*)state->mem64) + state->memsize, input, 32-state->memsize); + state->v1 = XXH64_round(state->v1, XXH_readLE64(state->mem64+0, endian)); + state->v2 = XXH64_round(state->v2, XXH_readLE64(state->mem64+1, endian)); + state->v3 = XXH64_round(state->v3, XXH_readLE64(state->mem64+2, endian)); + state->v4 = XXH64_round(state->v4, XXH_readLE64(state->mem64+3, endian)); + p += 32-state->memsize; + state->memsize = 0; + } + + if (p+32 <= bEnd) { + const BYTE* const limit = bEnd - 32; + U64 v1 = state->v1; + U64 v2 = state->v2; + U64 v3 = state->v3; + U64 v4 = state->v4; + + do { + v1 = XXH64_round(v1, XXH_readLE64(p, endian)); p+=8; + v2 = XXH64_round(v2, XXH_readLE64(p, endian)); p+=8; + v3 = XXH64_round(v3, XXH_readLE64(p, endian)); p+=8; + v4 = XXH64_round(v4, XXH_readLE64(p, endian)); p+=8; + } while (p<=limit); + + state->v1 = v1; + state->v2 = v2; + state->v3 = v3; + state->v4 = v4; + } + + if (p < bEnd) { + XXH_memcpy(state->mem64, p, (size_t)(bEnd-p)); + state->memsize = (unsigned)(bEnd-p); + } + + return XXH_OK; +} + +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* state_in, const void* input, size_t len) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_update_endian(state_in, input, len, XXH_littleEndian); + else + return XXH64_update_endian(state_in, input, len, XXH_bigEndian); +} + + + +FORCE_INLINE U64 XXH64_digest_endian (const XXH64_state_t* state, XXH_endianess endian) +{ + const BYTE * p = (const BYTE*)state->mem64; + const BYTE* const bEnd = (const BYTE*)state->mem64 + state->memsize; + U64 h64; + + if (state->total_len >= 32) { + U64 const v1 = state->v1; + U64 const v2 = state->v2; + U64 const v3 = state->v3; + U64 const v4 = state->v4; + + h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18); + h64 = XXH64_mergeRound(h64, v1); + h64 = XXH64_mergeRound(h64, v2); + h64 = XXH64_mergeRound(h64, v3); + h64 = XXH64_mergeRound(h64, v4); + } else { + h64 = state->v3 + PRIME64_5; + } + + h64 += (U64) state->total_len; + + while (p+8<=bEnd) { + U64 const k1 = XXH64_round(0, XXH_readLE64(p, endian)); + h64 ^= k1; + h64 = XXH_rotl64(h64,27) * PRIME64_1 + PRIME64_4; + p+=8; + } + + if (p+4<=bEnd) { + h64 ^= (U64)(XXH_readLE32(p, endian)) * PRIME64_1; + h64 = XXH_rotl64(h64, 23) * PRIME64_2 + PRIME64_3; + p+=4; + } + + while (p> 33; + h64 *= PRIME64_2; + h64 ^= h64 >> 29; + h64 *= PRIME64_3; + h64 ^= h64 >> 32; + + return h64; +} + + +XXH_PUBLIC_API unsigned long long XXH64_digest (const XXH64_state_t* state_in) +{ + XXH_endianess endian_detected = (XXH_endianess)XXH_CPU_LITTLE_ENDIAN; + + if ((endian_detected==XXH_littleEndian) || XXH_FORCE_NATIVE_FORMAT) + return XXH64_digest_endian(state_in, XXH_littleEndian); + else + return XXH64_digest_endian(state_in, XXH_bigEndian); +} + + +/* ************************** +* Canonical representation +****************************/ + +/*! Default XXH result types are basic unsigned 32 and 64 bits. +* The canonical representation follows human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file or buffer, and remain comparable across different systems and programs. +*/ + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash) +{ + XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t)); + if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash); + memcpy(dst, &hash, sizeof(*dst)); +} + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src) +{ + return XXH_readBE32(src); +} + +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src) +{ + return XXH_readBE64(src); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/xxhash.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/xxhash.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/xxhash.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/xxhash.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,309 @@ +/* + xxHash - Extremely Fast Hash algorithm + Header File + Copyright (C) 2012-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - xxHash source repository : https://github.com/Cyan4973/xxHash +*/ + +/* Notice extracted from xxHash homepage : + +xxHash is an extremely fast Hash algorithm, running at RAM speed limits. +It also successfully passes all tests from the SMHasher suite. + +Comparison (single thread, Windows Seven 32 bits, using SMHasher on a Core 2 Duo @3GHz) + +Name Speed Q.Score Author +xxHash 5.4 GB/s 10 +CrapWow 3.2 GB/s 2 Andrew +MumurHash 3a 2.7 GB/s 10 Austin Appleby +SpookyHash 2.0 GB/s 10 Bob Jenkins +SBox 1.4 GB/s 9 Bret Mulvey +Lookup3 1.2 GB/s 9 Bob Jenkins +SuperFastHash 1.2 GB/s 1 Paul Hsieh +CityHash64 1.05 GB/s 10 Pike & Alakuijala +FNV 0.55 GB/s 5 Fowler, Noll, Vo +CRC32 0.43 GB/s 9 +MD5-32 0.33 GB/s 10 Ronald L. Rivest +SHA1-32 0.28 GB/s 10 + +Q.Score is a measure of quality of the hash function. +It depends on successfully passing SMHasher test set. +10 is a perfect score. + +A 64-bits version, named XXH64, is available since r35. +It offers much better speed, but for 64-bits applications only. +Name Speed on 64 bits Speed on 32 bits +XXH64 13.8 GB/s 1.9 GB/s +XXH32 6.8 GB/s 6.0 GB/s +*/ + +#ifndef XXHASH_H_5627135585666179 +#define XXHASH_H_5627135585666179 1 + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef XXH_NAMESPACE +# define XXH_NAMESPACE ZSTD_ /* Zstandard specific */ +#endif + + +/* **************************** +* Definitions +******************************/ +#include /* size_t */ +typedef enum { XXH_OK=0, XXH_ERROR } XXH_errorcode; + + +/* **************************** +* API modifier +******************************/ +/** XXH_PRIVATE_API +* This is useful if you want to include xxhash functions in `static` mode +* in order to inline them, and remove their symbol from the public list. +* Methodology : +* #define XXH_PRIVATE_API +* #include "xxhash.h" +* `xxhash.c` is automatically included. +* It's not useful to compile and link it as a separate module anymore. +*/ +#ifdef XXH_PRIVATE_API +# ifndef XXH_STATIC_LINKING_ONLY +# define XXH_STATIC_LINKING_ONLY +# endif +# if defined(__GNUC__) +# define XXH_PUBLIC_API static __inline __attribute__((unused)) +# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +# define XXH_PUBLIC_API static inline +# elif defined(_MSC_VER) +# define XXH_PUBLIC_API static __inline +# else +# define XXH_PUBLIC_API static /* this version may generate warnings for unused static functions; disable the relevant warning */ +# endif +#else +# define XXH_PUBLIC_API /* do nothing */ +#endif /* XXH_PRIVATE_API */ + +/*!XXH_NAMESPACE, aka Namespace Emulation : + +If you want to include _and expose_ xxHash functions from within your own library, +but also want to avoid symbol collisions with another library which also includes xxHash, + +you can use XXH_NAMESPACE, to automatically prefix any public symbol from xxhash library +with the value of XXH_NAMESPACE (so avoid to keep it NULL and avoid numeric values). + +Note that no change is required within the calling program as long as it includes `xxhash.h` : +regular symbol name will be automatically translated by this header. +*/ +#ifdef XXH_NAMESPACE +# define XXH_CAT(A,B) A##B +# define XXH_NAME2(A,B) XXH_CAT(A,B) +# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32) +# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64) +# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber) +# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState) +# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState) +# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState) +# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState) +# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset) +# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset) +# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update) +# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update) +# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest) +# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest) +# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState) +# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState) +# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash) +# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash) +# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical) +# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical) +#endif + + +/* ************************************* +* Version +***************************************/ +#define XXH_VERSION_MAJOR 0 +#define XXH_VERSION_MINOR 6 +#define XXH_VERSION_RELEASE 2 +#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE) +XXH_PUBLIC_API unsigned XXH_versionNumber (void); + + +/* **************************** +* Simple Hash Functions +******************************/ +typedef unsigned int XXH32_hash_t; +typedef unsigned long long XXH64_hash_t; + +XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t length, unsigned int seed); +XXH_PUBLIC_API XXH64_hash_t XXH64 (const void* input, size_t length, unsigned long long seed); + +/*! +XXH32() : + Calculate the 32-bits hash of sequence "length" bytes stored at memory address "input". + The memory between input & input+length must be valid (allocated and read-accessible). + "seed" can be used to alter the result predictably. + Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark) : 5.4 GB/s +XXH64() : + Calculate the 64-bits hash of sequence of length "len" stored at memory address "input". + "seed" can be used to alter the result predictably. + This function runs 2x faster on 64-bits systems, but slower on 32-bits systems (see benchmark). +*/ + + +/* **************************** +* Streaming Hash Functions +******************************/ +typedef struct XXH32_state_s XXH32_state_t; /* incomplete type */ +typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */ + +/*! State allocation, compatible with dynamic libraries */ + +XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void); +XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr); + + +/* hash streaming */ + +XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, unsigned int seed); +XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr); + +XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH64_state_t* statePtr, unsigned long long seed); +XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH64_state_t* statePtr, const void* input, size_t length); +XXH_PUBLIC_API XXH64_hash_t XXH64_digest (const XXH64_state_t* statePtr); + +/* +These functions generate the xxHash of an input provided in multiple segments. +Note that, for small input, they are slower than single-call functions, due to state management. +For small input, prefer `XXH32()` and `XXH64()` . + +XXH state must first be allocated, using XXH*_createState() . + +Start a new hash by initializing state with a seed, using XXH*_reset(). + +Then, feed the hash state by calling XXH*_update() as many times as necessary. +Obviously, input must be allocated and read accessible. +The function returns an error code, with 0 meaning OK, and any other value meaning there is an error. + +Finally, a hash value can be produced anytime, by using XXH*_digest(). +This function returns the nn-bits hash as an int or long long. + +It's still possible to continue inserting input into the hash state after a digest, +and generate some new hashes later on, by calling again XXH*_digest(). + +When done, free XXH state space if it was allocated dynamically. +*/ + + +/* ************************** +* Utils +****************************/ +#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* ! C99 */ +# define restrict /* disable restrict */ +#endif + +XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* restrict dst_state, const XXH32_state_t* restrict src_state); +XXH_PUBLIC_API void XXH64_copyState(XXH64_state_t* restrict dst_state, const XXH64_state_t* restrict src_state); + + +/* ************************** +* Canonical representation +****************************/ +typedef struct { unsigned char digest[4]; } XXH32_canonical_t; +typedef struct { unsigned char digest[8]; } XXH64_canonical_t; + +XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash); +XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH64_canonical_t* dst, XXH64_hash_t hash); + +XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src); +XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(const XXH64_canonical_t* src); + +/* Default result type for XXH functions are primitive unsigned 32 and 64 bits. +* The canonical representation uses human-readable write convention, aka big-endian (large digits first). +* These functions allow transformation of hash result into and from its canonical format. +* This way, hash values can be written into a file / memory, and remain comparable on different systems and programs. +*/ + + +#ifdef XXH_STATIC_LINKING_ONLY + +/* ================================================================================================ + This section contains definitions which are not guaranteed to remain stable. + They may change in future versions, becoming incompatible with a different version of the library. + They shall only be used with static linking. + Never use these definitions in association with dynamic linking ! +=================================================================================================== */ + +/* These definitions are only meant to allow allocation of XXH state + statically, on stack, or in a struct for example. + Do not use members directly. */ + + struct XXH32_state_s { + unsigned total_len_32; + unsigned large_len; + unsigned v1; + unsigned v2; + unsigned v3; + unsigned v4; + unsigned mem32[4]; /* buffer defined as U32 for alignment */ + unsigned memsize; + unsigned reserved; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH32_state_t */ + + struct XXH64_state_s { + unsigned long long total_len; + unsigned long long v1; + unsigned long long v2; + unsigned long long v3; + unsigned long long v4; + unsigned long long mem64[4]; /* buffer defined as U64 for alignment */ + unsigned memsize; + unsigned reserved[2]; /* never read nor write, will be removed in a future version */ + }; /* typedef'd to XXH64_state_t */ + + +# ifdef XXH_PRIVATE_API +# include "xxhash.c" /* include xxhash functions as `static`, for inlining */ +# endif + +#endif /* XXH_STATIC_LINKING_ONLY */ + + +#if defined (__cplusplus) +} +#endif + +#endif /* XXHASH_H_5627135585666179 */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/zstd_common.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/zstd_common.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/zstd_common.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/zstd_common.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,77 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + + +/*-************************************* +* Dependencies +***************************************/ +#include /* malloc */ +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" /* declaration of ZSTD_isError, ZSTD_getErrorName, ZSTD_getErrorCode, ZSTD_getErrorString, ZSTD_versionNumber */ + + +/*-**************************************** +* Version +******************************************/ +unsigned ZSTD_versionNumber (void) { return ZSTD_VERSION_NUMBER; } + + +/*-**************************************** +* ZSTD Error Management +******************************************/ +/*! ZSTD_isError() : +* tells if a return value is an error code */ +unsigned ZSTD_isError(size_t code) { return ERR_isError(code); } + +/*! ZSTD_getErrorName() : +* provides error code string from function result (useful for debugging) */ +const char* ZSTD_getErrorName(size_t code) { return ERR_getErrorName(code); } + +/*! ZSTD_getError() : +* convert a `size_t` function result into a proper ZSTD_errorCode enum */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } + +/*! ZSTD_getErrorString() : +* provides error code string from enum */ +const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorName(code); } + +/* --- ZBUFF Error Management (deprecated) --- */ +unsigned ZBUFF_isError(size_t errorCode) { return ERR_isError(errorCode); } +const char* ZBUFF_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + + +/*=************************************************************** +* Custom allocator +****************************************************************/ +/* default uses stdlib */ +void* ZSTD_defaultAllocFunction(void* opaque, size_t size) +{ + void* address = malloc(size); + (void)opaque; + return address; +} + +void ZSTD_defaultFreeFunction(void* opaque, void* address) +{ + (void)opaque; + free(address); +} + +void* ZSTD_malloc(size_t size, ZSTD_customMem customMem) +{ + return customMem.customAlloc(customMem.opaque, size); +} + +void ZSTD_free(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) + customMem.customFree(customMem.opaque, ptr); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/zstd_errors.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/zstd_errors.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/zstd_errors.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/zstd_errors.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,60 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef ZSTD_ERRORS_H_398273423 +#define ZSTD_ERRORS_H_398273423 + +#if defined (__cplusplus) +extern "C" { +#endif + +/*===== dependency =====*/ +#include /* size_t */ + + +/*-**************************************** +* error codes list +******************************************/ +typedef enum { + ZSTD_error_no_error, + ZSTD_error_GENERIC, + ZSTD_error_prefix_unknown, + ZSTD_error_version_unsupported, + ZSTD_error_parameter_unknown, + ZSTD_error_frameParameter_unsupported, + ZSTD_error_frameParameter_unsupportedBy32bits, + ZSTD_error_frameParameter_windowTooLarge, + ZSTD_error_compressionParameter_unsupported, + ZSTD_error_init_missing, + ZSTD_error_memory_allocation, + ZSTD_error_stage_wrong, + ZSTD_error_dstSize_tooSmall, + ZSTD_error_srcSize_wrong, + ZSTD_error_corruption_detected, + ZSTD_error_checksum_wrong, + ZSTD_error_tableLog_tooLarge, + ZSTD_error_maxSymbolValue_tooLarge, + ZSTD_error_maxSymbolValue_tooSmall, + ZSTD_error_dictionary_corrupted, + ZSTD_error_dictionary_wrong, + ZSTD_error_maxCode +} ZSTD_ErrorCode; + +/*! ZSTD_getErrorCode() : + convert a `size_t` function result into a `ZSTD_ErrorCode` enum type, + which can be used to compare directly with enum list published into "error_public.h" */ +ZSTD_ErrorCode ZSTD_getErrorCode(size_t functionResult); +const char* ZSTD_getErrorString(ZSTD_ErrorCode code); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ERRORS_H_398273423 */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/zstd_internal.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/zstd_internal.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/common/zstd_internal.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/common/zstd_internal.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,270 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef ZSTD_CCOMMON_H_MODULE +#define ZSTD_CCOMMON_H_MODULE + +/*-******************************************************* +* Compiler specifics +*********************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4324) /* disable: C4324: padded structure */ +# pragma warning(disable : 4100) /* disable: C4100: unreferenced formal parameter */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + +#ifdef _MSC_VER +# define FORCE_NOINLINE static __declspec(noinline) +#else +# ifdef __GNUC__ +# define FORCE_NOINLINE static __attribute__((__noinline__)) +# else +# define FORCE_NOINLINE static +# endif +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include "mem.h" +#include "error_private.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + + +/*-************************************* +* shared macros +***************************************/ +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) +#define CHECK_F(f) { size_t const errcod = f; if (ERR_isError(errcod)) return errcod; } /* check and Forward error code */ +#define CHECK_E(f, e) { size_t const errcod = f; if (ERR_isError(errcod)) return ERROR(e); } /* check and send Error code */ + + +/*-************************************* +* Common constants +***************************************/ +#define ZSTD_OPT_NUM (1<<12) +#define ZSTD_DICT_MAGIC 0xEC30A437 /* v0.7+ */ + +#define ZSTD_REP_NUM 3 /* number of repcodes */ +#define ZSTD_REP_CHECK (ZSTD_REP_NUM) /* number of repcodes to check by the optimal parser */ +#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) +#define ZSTD_REP_MOVE_OPT (ZSTD_REP_NUM) +static const U32 repStartValue[ZSTD_REP_NUM] = { 1, 4, 8 }; + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define BIT7 128 +#define BIT6 64 +#define BIT5 32 +#define BIT4 16 +#define BIT1 2 +#define BIT0 1 + +#define ZSTD_WINDOWLOG_ABSOLUTEMIN 10 +static const size_t ZSTD_fcs_fieldSize[4] = { 0, 2, 4, 8 }; +static const size_t ZSTD_did_fieldSize[4] = { 0, 1, 2, 4 }; + +#define ZSTD_BLOCKHEADERSIZE 3 /* C standard doesn't allow `static const` variable to be init using another `static const` variable */ +static const size_t ZSTD_blockHeaderSize = ZSTD_BLOCKHEADERSIZE; +typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; + +#define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ + +#define HufLog 12 +typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; + +#define LONGNBSEQ 0x7F00 + +#define MINMATCH 3 +#define EQUAL_READ32 4 + +#define Litbits 8 +#define MaxLit ((1<= 3) /* GCC Intrinsic */ + return 31 - __builtin_clz(val); +# else /* Software version */ + static const int DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; + U32 v = val; + int r; + v |= v >> 1; + v |= v >> 2; + v |= v >> 4; + v |= v >> 8; + v |= v >> 16; + r = DeBruijnClz[(U32)(v * 0x07C4ACDDU) >> 27]; + return r; +# endif +} + + +#endif /* ZSTD_CCOMMON_H_MODULE */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/fse_compress.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/fse_compress.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/fse_compress.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/fse_compress.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,850 @@ +/* ****************************************************************** + FSE : Finite State Entropy encoder + Copyright (C) 2013-2015, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# define FORCE_INLINE static __forceinline +# include /* For Visual 2005 */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +# pragma warning(disable : 4214) /* disable: C4214: non-int bitfields */ +#else +# if defined (__cplusplus) || defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* C99 */ +# ifdef __GNUC__ +# define FORCE_INLINE static inline __attribute__((always_inline)) +# else +# define FORCE_INLINE static inline +# endif +# else +# define FORCE_INLINE static +# endif /* __STDC_VERSION__ */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* malloc, free, qsort */ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define FSE_STATIC_ASSERT(c) { enum { FSE_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/* ************************************************************** +* Templates +****************************************************************/ +/* + designed to be included + for type-specific functions (template emulation in C) + Objective is to write these functions only once, for improved maintenance +*/ + +/* safety checks */ +#ifndef FSE_FUNCTION_EXTENSION +# error "FSE_FUNCTION_EXTENSION must be defined" +#endif +#ifndef FSE_FUNCTION_TYPE +# error "FSE_FUNCTION_TYPE must be defined" +#endif + +/* Function names */ +#define FSE_CAT(X,Y) X##Y +#define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) +#define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) + + +/* Function templates */ + +/* FSE_buildCTable_wksp() : + * Same as FSE_buildCTable(), but using an externally allocated scratch buffer (`workSpace`). + * wkspSize should be sized to handle worst case situation, which is `1<>1 : 1) ; + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + U32 const step = FSE_TABLESTEP(tableSize); + U32 cumul[FSE_MAX_SYMBOL_VALUE+2]; + + FSE_FUNCTION_TYPE* const tableSymbol = (FSE_FUNCTION_TYPE*)workSpace; + U32 highThreshold = tableSize-1; + + /* CTable header */ + if (((size_t)1 << tableLog) * sizeof(FSE_FUNCTION_TYPE) > wkspSize) return ERROR(tableLog_tooLarge); + tableU16[-2] = (U16) tableLog; + tableU16[-1] = (U16) maxSymbolValue; + + /* For explanations on how to distribute symbol values over the table : + * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + + /* symbol start positions */ + { U32 u; + cumul[0] = 0; + for (u=1; u<=maxSymbolValue+1; u++) { + if (normalizedCounter[u-1]==-1) { /* Low proba symbol */ + cumul[u] = cumul[u-1] + 1; + tableSymbol[highThreshold--] = (FSE_FUNCTION_TYPE)(u-1); + } else { + cumul[u] = cumul[u-1] + normalizedCounter[u-1]; + } } + cumul[maxSymbolValue+1] = tableSize+1; + } + + /* Spread symbols */ + { U32 position = 0; + U32 symbol; + for (symbol=0; symbol<=maxSymbolValue; symbol++) { + int nbOccurences; + for (nbOccurences=0; nbOccurences highThreshold) position = (position + step) & tableMask; /* Low proba area */ + } } + + if (position!=0) return ERROR(GENERIC); /* Must have gone through all positions */ + } + + /* Build table */ + { U32 u; for (u=0; u> 3) + 3; + return maxSymbolValue ? maxHeaderSize : FSE_NCOUNTBOUND; /* maxSymbolValue==0 ? use default */ +} + +static short FSE_abs(short a) { return (short)(a<0 ? -a : a); } + +static size_t FSE_writeNCount_generic (void* header, size_t headerBufferSize, + const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, + unsigned writeIsSafe) +{ + BYTE* const ostart = (BYTE*) header; + BYTE* out = ostart; + BYTE* const oend = ostart + headerBufferSize; + int nbBits; + const int tableSize = 1 << tableLog; + int remaining; + int threshold; + U32 bitStream; + int bitCount; + unsigned charnum = 0; + int previous0 = 0; + + bitStream = 0; + bitCount = 0; + /* Table Size */ + bitStream += (tableLog-FSE_MIN_TABLELOG) << bitCount; + bitCount += 4; + + /* Init */ + remaining = tableSize+1; /* +1 for extra accuracy */ + threshold = tableSize; + nbBits = tableLog+1; + + while (remaining>1) { /* stops at 1 */ + if (previous0) { + unsigned start = charnum; + while (!normalizedCounter[charnum]) charnum++; + while (charnum >= start+24) { + start+=24; + bitStream += 0xFFFFU << bitCount; + if ((!writeIsSafe) && (out > oend-2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE) bitStream; + out[1] = (BYTE)(bitStream>>8); + out+=2; + bitStream>>=16; + } + while (charnum >= start+3) { + start+=3; + bitStream += 3 << bitCount; + bitCount += 2; + } + bitStream += (charnum-start) << bitCount; + bitCount += 2; + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + { short count = normalizedCounter[charnum++]; + const short max = (short)((2*threshold-1)-remaining); + remaining -= FSE_abs(count); + if (remaining<1) return ERROR(GENERIC); + count++; /* +1 for extra accuracy */ + if (count>=threshold) count += max; /* [0..max[ [max..threshold[ (...) [threshold+max 2*threshold[ */ + bitStream += count << bitCount; + bitCount += nbBits; + bitCount -= (count>=1; + } + if (bitCount>16) { + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out += 2; + bitStream >>= 16; + bitCount -= 16; + } } + + /* flush remaining bitStream */ + if ((!writeIsSafe) && (out > oend - 2)) return ERROR(dstSize_tooSmall); /* Buffer overflow */ + out[0] = (BYTE)bitStream; + out[1] = (BYTE)(bitStream>>8); + out+= (bitCount+7) /8; + + if (charnum > maxSymbolValue + 1) return ERROR(GENERIC); + + return (out-ostart); +} + + +size_t FSE_writeNCount (void* buffer, size_t bufferSize, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported */ + + if (bufferSize < FSE_NCountWriteBound(maxSymbolValue, tableLog)) + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 0); + + return FSE_writeNCount_generic(buffer, bufferSize, normalizedCounter, maxSymbolValue, tableLog, 1); +} + + + +/*-************************************************************** +* Counting histogram +****************************************************************/ +/*! FSE_count_simple + This function counts byte values within `src`, and store the histogram into table `count`. + It doesn't use any additional memory. + But this function is unsafe : it doesn't check that all values within `src` can fit into `count`. + For this reason, prefer using a table `count` with 256 elements. + @return : count of most numerous element +*/ +size_t FSE_count_simple(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + const BYTE* const end = ip + srcSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + + memset(count, 0, (maxSymbolValue+1)*sizeof(*count)); + if (srcSize==0) { *maxSymbolValuePtr = 0; return 0; } + + while (ip max) max = count[s]; } + + return (size_t)max; +} + + +/* FSE_count_parallel_wksp() : + * Same as FSE_count_parallel(), but using an externally provided scratch buffer. + * `workSpace` size must be a minimum of `1024 * sizeof(unsigned)`` */ +static size_t FSE_count_parallel_wksp( + unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, + unsigned checkMax, unsigned* const workSpace) +{ + const BYTE* ip = (const BYTE*)source; + const BYTE* const iend = ip+sourceSize; + unsigned maxSymbolValue = *maxSymbolValuePtr; + unsigned max=0; + U32* const Counting1 = workSpace; + U32* const Counting2 = Counting1 + 256; + U32* const Counting3 = Counting2 + 256; + U32* const Counting4 = Counting3 + 256; + + memset(Counting1, 0, 4*256*sizeof(unsigned)); + + /* safety checks */ + if (!sourceSize) { + memset(count, 0, maxSymbolValue + 1); + *maxSymbolValuePtr = 0; + return 0; + } + if (!maxSymbolValue) maxSymbolValue = 255; /* 0 == default */ + + /* by stripes of 16 bytes */ + { U32 cached = MEM_read32(ip); ip += 4; + while (ip < iend-15) { + U32 c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + c = cached; cached = MEM_read32(ip); ip += 4; + Counting1[(BYTE) c ]++; + Counting2[(BYTE)(c>>8) ]++; + Counting3[(BYTE)(c>>16)]++; + Counting4[ c>>24 ]++; + } + ip-=4; + } + + /* finish last symbols */ + while (ipmaxSymbolValue; s--) { + Counting1[s] += Counting2[s] + Counting3[s] + Counting4[s]; + if (Counting1[s]) return ERROR(maxSymbolValue_tooSmall); + } } + + { U32 s; for (s=0; s<=maxSymbolValue; s++) { + count[s] = Counting1[s] + Counting2[s] + Counting3[s] + Counting4[s]; + if (count[s] > max) max = count[s]; + } } + + while (!count[maxSymbolValue]) maxSymbolValue--; + *maxSymbolValuePtr = maxSymbolValue; + return (size_t)max; +} + +/* FSE_countFast_wksp() : + * Same as FSE_countFast(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= `1024` unsigned */ +size_t FSE_countFast_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace) +{ + if (sourceSize < 1500) return FSE_count_simple(count, maxSymbolValuePtr, source, sourceSize); + return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 0, workSpace); +} + +/* fast variant (unsafe : won't check if src contains values beyond count[] limit) */ +size_t FSE_countFast(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize) +{ + unsigned tmpCounters[1024]; + return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, tmpCounters); +} + +/* FSE_count_wksp() : + * Same as FSE_count(), but using an externally provided scratch buffer. + * `workSpace` size must be table of >= `1024` unsigned */ +size_t FSE_count_wksp(unsigned* count, unsigned* maxSymbolValuePtr, + const void* source, size_t sourceSize, unsigned* workSpace) +{ + if (*maxSymbolValuePtr < 255) + return FSE_count_parallel_wksp(count, maxSymbolValuePtr, source, sourceSize, 1, workSpace); + *maxSymbolValuePtr = 255; + return FSE_countFast_wksp(count, maxSymbolValuePtr, source, sourceSize, workSpace); +} + +size_t FSE_count(unsigned* count, unsigned* maxSymbolValuePtr, + const void* src, size_t srcSize) +{ + unsigned tmpCounters[1024]; + return FSE_count_wksp(count, maxSymbolValuePtr, src, srcSize, tmpCounters); +} + + + +/*-************************************************************** +* FSE Compression Code +****************************************************************/ +/*! FSE_sizeof_CTable() : + FSE_CTable is a variable size structure which contains : + `U16 tableLog;` + `U16 maxSymbolValue;` + `U16 nextStateNumber[1 << tableLog];` // This size is variable + `FSE_symbolCompressionTransform symbolTT[maxSymbolValue+1];` // This size is variable +Allocation is manual (C standard does not support variable-size structures). +*/ +size_t FSE_sizeof_CTable (unsigned maxSymbolValue, unsigned tableLog) +{ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); +} + +FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) +{ + size_t size; + if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; + size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); + return (FSE_CTable*)malloc(size); +} + +void FSE_freeCTable (FSE_CTable* ct) { free(ct); } + +/* provides the minimum logSize to safely represent a distribution */ +static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) +{ + U32 minBitsSrc = BIT_highbit32((U32)(srcSize - 1)) + 1; + U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; + return minBits; +} + +unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) +{ + U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 tableLog = maxTableLog; + U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (maxBitsSrc < tableLog) tableLog = maxBitsSrc; /* Accuracy can be reduced */ + if (minBits > tableLog) tableLog = minBits; /* Need a minimum to safely represent all symbol values */ + if (tableLog < FSE_MIN_TABLELOG) tableLog = FSE_MIN_TABLELOG; + if (tableLog > FSE_MAX_TABLELOG) tableLog = FSE_MAX_TABLELOG; + return tableLog; +} + +unsigned FSE_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 2); +} + + +/* Secondary normalization method. + To be used when primary method fails. */ + +static size_t FSE_normalizeM2(short* norm, U32 tableLog, const unsigned* count, size_t total, U32 maxSymbolValue) +{ + U32 s; + U32 distributed = 0; + U32 ToDistribute; + + /* Init */ + U32 const lowThreshold = (U32)(total >> tableLog); + U32 lowOne = (U32)((total * 3) >> (tableLog + 1)); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == 0) { + norm[s]=0; + continue; + } + if (count[s] <= lowThreshold) { + norm[s] = -1; + distributed++; + total -= count[s]; + continue; + } + if (count[s] <= lowOne) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } + norm[s]=-2; + } + ToDistribute = (1 << tableLog) - distributed; + + if ((total / ToDistribute) > lowOne) { + /* risk of rounding to zero */ + lowOne = (U32)((total * 3) / (ToDistribute * 2)); + for (s=0; s<=maxSymbolValue; s++) { + if ((norm[s] == -2) && (count[s] <= lowOne)) { + norm[s] = 1; + distributed++; + total -= count[s]; + continue; + } } + ToDistribute = (1 << tableLog) - distributed; + } + + if (distributed == maxSymbolValue+1) { + /* all values are pretty poor; + probably incompressible data (should have already been detected); + find max, then give all remaining points to max */ + U32 maxV = 0, maxC = 0; + for (s=0; s<=maxSymbolValue; s++) + if (count[s] > maxC) maxV=s, maxC=count[s]; + norm[maxV] += (short)ToDistribute; + return 0; + } + + { U64 const vStepLog = 62 - tableLog; + U64 const mid = (1ULL << (vStepLog-1)) - 1; + U64 const rStep = ((((U64)1<> vStepLog); + U32 const sEnd = (U32)(end >> vStepLog); + U32 const weight = sEnd - sStart; + if (weight < 1) + return ERROR(GENERIC); + norm[s] = (short)weight; + tmpTotal = end; + } } } + + return 0; +} + + +size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, + const unsigned* count, size_t total, + unsigned maxSymbolValue) +{ + /* Sanity checks */ + if (tableLog==0) tableLog = FSE_DEFAULT_TABLELOG; + if (tableLog < FSE_MIN_TABLELOG) return ERROR(GENERIC); /* Unsupported size */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); /* Unsupported size */ + if (tableLog < FSE_minTableLog(total, maxSymbolValue)) return ERROR(GENERIC); /* Too small tableLog, compression potentially impossible */ + + { U32 const rtbTable[] = { 0, 473195, 504333, 520860, 550000, 700000, 750000, 830000 }; + U64 const scale = 62 - tableLog; + U64 const step = ((U64)1<<62) / total; /* <== here, one division ! */ + U64 const vStep = 1ULL<<(scale-20); + int stillToDistribute = 1<> tableLog); + + for (s=0; s<=maxSymbolValue; s++) { + if (count[s] == total) return 0; /* rle special case */ + if (count[s] == 0) { normalizedCounter[s]=0; continue; } + if (count[s] <= lowThreshold) { + normalizedCounter[s] = -1; + stillToDistribute--; + } else { + short proba = (short)((count[s]*step) >> scale); + if (proba<8) { + U64 restToBeat = vStep * rtbTable[proba]; + proba += (count[s]*step) - ((U64)proba< restToBeat; + } + if (proba > largestP) largestP=proba, largest=s; + normalizedCounter[s] = proba; + stillToDistribute -= proba; + } } + if (-stillToDistribute >= (normalizedCounter[largest] >> 1)) { + /* corner case, need another normalization method */ + size_t const errorCode = FSE_normalizeM2(normalizedCounter, tableLog, count, total, maxSymbolValue); + if (FSE_isError(errorCode)) return errorCode; + } + else normalizedCounter[largest] += (short)stillToDistribute; + } + +#if 0 + { /* Print Table (debug) */ + U32 s; + U32 nTotal = 0; + for (s=0; s<=maxSymbolValue; s++) + printf("%3i: %4i \n", s, normalizedCounter[s]); + for (s=0; s<=maxSymbolValue; s++) + nTotal += abs(normalizedCounter[s]); + if (nTotal != (1U<>1); /* assumption : tableLog >= 1 */ + FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); + unsigned s; + + /* Sanity checks */ + if (nbBits < 1) return ERROR(GENERIC); /* min size */ + + /* header */ + tableU16[-2] = (U16) nbBits; + tableU16[-1] = (U16) maxSymbolValue; + + /* Build table */ + for (s=0; s FSE_MAX_TABLELOG*4+7 ) && (srcSize & 2)) { /* test bit 2 */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + FSE_FLUSHBITS(&bitC); + } + + /* 2 or 4 encoding per loop */ + while ( ip>istart ) { + + FSE_encodeSymbol(&bitC, &CState2, *--ip); + + if (sizeof(bitC.bitContainer)*8 < FSE_MAX_TABLELOG*2+7 ) /* this test must be static */ + FSE_FLUSHBITS(&bitC); + + FSE_encodeSymbol(&bitC, &CState1, *--ip); + + if (sizeof(bitC.bitContainer)*8 > FSE_MAX_TABLELOG*4+7 ) { /* this test must be static */ + FSE_encodeSymbol(&bitC, &CState2, *--ip); + FSE_encodeSymbol(&bitC, &CState1, *--ip); + } + + FSE_FLUSHBITS(&bitC); + } + + FSE_flushCState(&bitC, &CState2); + FSE_flushCState(&bitC, &CState1); + return BIT_closeCStream(&bitC); +} + +size_t FSE_compress_usingCTable (void* dst, size_t dstSize, + const void* src, size_t srcSize, + const FSE_CTable* ct) +{ + unsigned const fast = (dstSize >= FSE_BLOCKBOUND(srcSize)); + + if (fast) + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 1); + else + return FSE_compress_usingCTable_generic(dst, dstSize, src, srcSize, ct, 0); +} + + +size_t FSE_compressBound(size_t size) { return FSE_COMPRESSBOUND(size); } + +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + +/* FSE_compress_wksp() : + * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). + * `wkspSize` size must be `(1< not compressible */ + if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ + } + + tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue) ); + + /* Write table description header */ + { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += nc_err; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + /* check compressibility */ + if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; + + return op-ostart; +} + +typedef struct { + FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; + BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; +} fseWkspMax_t; + +size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) +{ + fseWkspMax_t scratchBuffer; + FSE_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ + if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); + return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); +} + +size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); +} + + +#endif /* FSE_COMMONDEFS_ONLY */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/huf_compress.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/huf_compress.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/huf_compress.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/huf_compress.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,609 @@ +/* ****************************************************************** + Huffman encoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Includes +****************************************************************/ +#include /* memcpy, memset */ +#include /* printf (debug) */ +#include "bitstream.h" +#define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ +#define CHECK_V_F(e, f) size_t const e = f; if (ERR_isError(e)) return f +#define CHECK_F(f) { CHECK_V_F(_var_err__, f); } + + +/* ************************************************************** +* Utils +****************************************************************/ +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +{ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); +} + + +/* ******************************************************* +* HUF : Huffman block compression +*********************************************************/ +/* HUF_compressWeights() : + * Same as FSE_compress(), but dedicated to huff0's weights compression. + * The use case needs much less stack memory. + * Note : all elements within weightTable are supposed to be <= HUF_TABLELOG_MAX. + */ +#define MAX_FSE_TABLELOG_FOR_HUFF_HEADER 6 +size_t HUF_compressWeights (void* dst, size_t dstSize, const void* weightTable, size_t wtSize) +{ + BYTE* const ostart = (BYTE*) dst; + BYTE* op = ostart; + BYTE* const oend = ostart + dstSize; + + U32 maxSymbolValue = HUF_TABLELOG_MAX; + U32 tableLog = MAX_FSE_TABLELOG_FOR_HUFF_HEADER; + + FSE_CTable CTable[FSE_CTABLE_SIZE_U32(MAX_FSE_TABLELOG_FOR_HUFF_HEADER, HUF_TABLELOG_MAX)]; + BYTE scratchBuffer[1< not compressible */ + } + + tableLog = FSE_optimalTableLog(tableLog, wtSize, maxSymbolValue); + CHECK_F( FSE_normalizeCount(norm, tableLog, count, wtSize, maxSymbolValue) ); + + /* Write table description header */ + { CHECK_V_F(hSize, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); + op += hSize; + } + + /* Compress */ + CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, sizeof(scratchBuffer)) ); + { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, weightTable, wtSize, CTable) ); + if (cSize == 0) return 0; /* not enough space for compressed data */ + op += cSize; + } + + return op-ostart; +} + + +struct HUF_CElt_s { + U16 val; + BYTE nbBits; +}; /* typedef'd to HUF_CElt within "huf.h" */ + +/*! HUF_writeCTable() : + `CTable` : huffman tree to save, using huf representation. + @return : size of saved CTable */ +size_t HUF_writeCTable (void* dst, size_t maxDstSize, + const HUF_CElt* CTable, U32 maxSymbolValue, U32 huffLog) +{ + BYTE bitsToWeight[HUF_TABLELOG_MAX + 1]; /* precomputed conversion table */ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX]; + BYTE* op = (BYTE*)dst; + U32 n; + + /* check conditions */ + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); + + /* convert to weight */ + bitsToWeight[0] = 0; + for (n=1; n1) & (hSize < maxSymbolValue/2)) { /* FSE compressed */ + op[0] = (BYTE)hSize; + return hSize+1; + } } + + /* write raw values as 4-bits (max : 15) */ + if (maxSymbolValue > (256-128)) return ERROR(GENERIC); /* should not happen : likely means source cannot be compressed */ + if (((maxSymbolValue+1)/2) + 1 > maxDstSize) return ERROR(dstSize_tooSmall); /* not enough space within dst buffer */ + op[0] = (BYTE)(128 /*special case*/ + (maxSymbolValue-1)); + huffWeight[maxSymbolValue] = 0; /* to be sure it doesn't cause msan issue in final combination */ + for (n=0; n HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (nbSymbols > maxSymbolValue+1) return ERROR(maxSymbolValue_tooSmall); + + /* Prepare base value per rank */ + { U32 n, nextRankStart = 0; + for (n=1; n<=tableLog; n++) { + U32 current = nextRankStart; + nextRankStart += (rankVal[n] << (n-1)); + rankVal[n] = current; + } } + + /* fill nbBits */ + { U32 n; for (n=0; nn=tableLog+1 */ + U16 valPerRank[HUF_TABLELOG_MAX+2] = {0}; + { U32 n; for (n=0; n0; n--) { /* start at n=tablelog <-> w=1 */ + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + /* assign value within rank, symbol order */ + { U32 n; for (n=0; n<=maxSymbolValue; n++) CTable[n].val = valPerRank[CTable[n].nbBits]++; } + } + + return readSize; +} + + +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +{ + const U32 largestBits = huffNode[lastNonNull].nbBits; + if (largestBits <= maxNbBits) return largestBits; /* early exit : no elt > maxNbBits */ + + /* there are several too large elements (at least >= 2) */ + { int totalCost = 0; + const U32 baseCost = 1 << (largestBits - maxNbBits); + U32 n = lastNonNull; + + while (huffNode[n].nbBits > maxNbBits) { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); + huffNode[n].nbBits = (BYTE)maxNbBits; + n --; + } /* n stops at huffNode[n].nbBits <= maxNbBits */ + while (huffNode[n].nbBits == maxNbBits) n--; /* n end at index of smallest symbol using < maxNbBits */ + + /* renorm totalCost */ + totalCost >>= (largestBits - maxNbBits); /* note : totalCost is necessarily a multiple of baseCost */ + + /* repay normalized cost */ + { U32 const noSymbol = 0xF0F0F0F0; + U32 rankLast[HUF_TABLELOG_MAX+2]; + int pos; + + /* Get pos of last (smallest) symbol per rank */ + memset(rankLast, 0xF0, sizeof(rankLast)); + { U32 currentNbBits = maxNbBits; + for (pos=n ; pos >= 0; pos--) { + if (huffNode[pos].nbBits >= currentNbBits) continue; + currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ + rankLast[maxNbBits-currentNbBits] = pos; + } } + + while (totalCost > 0) { + U32 nBitsToDecrease = BIT_highbit32(totalCost) + 1; + for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { + U32 highPos = rankLast[nBitsToDecrease]; + U32 lowPos = rankLast[nBitsToDecrease-1]; + if (highPos == noSymbol) continue; + if (lowPos == noSymbol) break; + { U32 const highTotal = huffNode[highPos].count; + U32 const lowTotal = 2 * huffNode[lowPos].count; + if (highTotal <= lowTotal) break; + } } + /* only triggered when no more rank 1 symbol left => find closest one (note : there is necessarily at least one !) */ + while ((nBitsToDecrease<=HUF_TABLELOG_MAX) && (rankLast[nBitsToDecrease] == noSymbol)) /* HUF_MAX_TABLELOG test just to please gcc 5+; but it should not be necessary */ + nBitsToDecrease ++; + totalCost -= 1 << (nBitsToDecrease-1); + if (rankLast[nBitsToDecrease-1] == noSymbol) + rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease]; /* this rank is no longer empty */ + huffNode[rankLast[nBitsToDecrease]].nbBits ++; + if (rankLast[nBitsToDecrease] == 0) /* special case, reached largest symbol */ + rankLast[nBitsToDecrease] = noSymbol; + else { + rankLast[nBitsToDecrease]--; + if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ + } } /* while (totalCost > 0) */ + + while (totalCost < 0) { /* Sometimes, cost correction overshoot */ + if (rankLast[1] == noSymbol) { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ + while (huffNode[n].nbBits == maxNbBits) n--; + huffNode[n+1].nbBits--; + rankLast[1] = n+1; + totalCost++; + continue; + } + huffNode[ rankLast[1] + 1 ].nbBits--; + rankLast[1]++; + totalCost ++; + } } } /* there are several too large elements (at least >= 2) */ + + return maxNbBits; +} + + +typedef struct { + U32 base; + U32 current; +} rankPos; + +static void HUF_sort(nodeElt* huffNode, const U32* count, U32 maxSymbolValue) +{ + rankPos rank[32]; + U32 n; + + memset(rank, 0, sizeof(rank)); + for (n=0; n<=maxSymbolValue; n++) { + U32 r = BIT_highbit32(count[n] + 1); + rank[r].base ++; + } + for (n=30; n>0; n--) rank[n-1].base += rank[n].base; + for (n=0; n<32; n++) rank[n].current = rank[n].base; + for (n=0; n<=maxSymbolValue; n++) { + U32 const c = count[n]; + U32 const r = BIT_highbit32(c+1) + 1; + U32 pos = rank[r].current++; + while ((pos > rank[r].base) && (c > huffNode[pos-1].count)) huffNode[pos]=huffNode[pos-1], pos--; + huffNode[pos].count = c; + huffNode[pos].byte = (BYTE)n; + } +} + + +/** HUF_buildCTable_wksp() : + * Same as HUF_buildCTable(), but using externally allocated scratch buffer. + * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as a table of 1024 unsigned. + */ +#define STARTNODE (HUF_SYMBOLVALUE_MAX+1) +typedef nodeElt huffNodeTable[2*HUF_SYMBOLVALUE_MAX+1 +1]; +size_t HUF_buildCTable_wksp (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +{ + nodeElt* const huffNode0 = (nodeElt*)workSpace; + nodeElt* const huffNode = huffNode0+1; + U32 n, nonNullRank; + int lowS, lowN; + U16 nodeNb = STARTNODE; + U32 nodeRoot; + + /* safety checks */ + if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); /* workSpace is not large enough */ + if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; + if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(GENERIC); + memset(huffNode0, 0, sizeof(huffNodeTable)); + + /* sort, decreasing order */ + HUF_sort(huffNode, count, maxSymbolValue); + + /* init for parents */ + nonNullRank = maxSymbolValue; + while(huffNode[nonNullRank].count == 0) nonNullRank--; + lowS = nonNullRank; nodeRoot = nodeNb + lowS - 1; lowN = nodeNb; + huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count; + huffNode[lowS].parent = huffNode[lowS-1].parent = nodeNb; + nodeNb++; lowS-=2; + for (n=nodeNb; n<=nodeRoot; n++) huffNode[n].count = (U32)(1U<<30); + huffNode0[0].count = (U32)(1U<<31); /* fake entry, strong barrier */ + + /* create parents */ + while (nodeNb <= nodeRoot) { + U32 n1 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + U32 n2 = (huffNode[lowS].count < huffNode[lowN].count) ? lowS-- : lowN++; + huffNode[nodeNb].count = huffNode[n1].count + huffNode[n2].count; + huffNode[n1].parent = huffNode[n2].parent = nodeNb; + nodeNb++; + } + + /* distribute weights (unlimited tree height) */ + huffNode[nodeRoot].nbBits = 0; + for (n=nodeRoot-1; n>=STARTNODE; n--) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + for (n=0; n<=nonNullRank; n++) + huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + + /* enforce maxTableLog */ + maxNbBits = HUF_setMaxHeight(huffNode, nonNullRank, maxNbBits); + + /* fill result into tree (val, nbBits) */ + { U16 nbPerRank[HUF_TABLELOG_MAX+1] = {0}; + U16 valPerRank[HUF_TABLELOG_MAX+1] = {0}; + if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ + for (n=0; n<=nonNullRank; n++) + nbPerRank[huffNode[n].nbBits]++; + /* determine stating value per rank */ + { U16 min = 0; + for (n=maxNbBits; n>0; n--) { + valPerRank[n] = min; /* get starting value within each rank */ + min += nbPerRank[n]; + min >>= 1; + } } + for (n=0; n<=maxSymbolValue; n++) + tree[huffNode[n].byte].nbBits = huffNode[n].nbBits; /* push nbBits per symbol, symbol order */ + for (n=0; n<=maxSymbolValue; n++) + tree[n].val = valPerRank[tree[n].nbBits]++; /* assign value within rank, symbol order */ + } + + return maxNbBits; +} + +/** HUF_buildCTable() : + * Note : count is used before tree is written, so they can safely overlap + */ +size_t HUF_buildCTable (HUF_CElt* tree, const U32* count, U32 maxSymbolValue, U32 maxNbBits) +{ + huffNodeTable nodeTable; + return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, nodeTable, sizeof(nodeTable)); +} + +static void HUF_encodeSymbol(BIT_CStream_t* bitCPtr, U32 symbol, const HUF_CElt* CTable) +{ + BIT_addBitsFast(bitCPtr, CTable[symbol].val, CTable[symbol].nbBits); +} + +size_t HUF_compressBound(size_t size) { return HUF_COMPRESSBOUND(size); } + +#define HUF_FLUSHBITS(s) (fast ? BIT_flushBitsFast(s) : BIT_flushBits(s)) + +#define HUF_FLUSHBITS_1(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*2+7) HUF_FLUSHBITS(stream) + +#define HUF_FLUSHBITS_2(stream) \ + if (sizeof((stream)->bitContainer)*8 < HUF_TABLELOG_MAX*4+7) HUF_FLUSHBITS(stream) + +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + const BYTE* ip = (const BYTE*) src; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + size_t n; + const unsigned fast = (dstSize >= HUF_BLOCKBOUND(srcSize)); + BIT_CStream_t bitC; + + /* init */ + if (dstSize < 8) return 0; /* not enough space to compress */ + { size_t const initErr = BIT_initCStream(&bitC, op, oend-op); + if (HUF_isError(initErr)) return 0; } + + n = srcSize & ~3; /* join to mod 4 */ + switch (srcSize & 3) + { + case 3 : HUF_encodeSymbol(&bitC, ip[n+ 2], CTable); + HUF_FLUSHBITS_2(&bitC); + case 2 : HUF_encodeSymbol(&bitC, ip[n+ 1], CTable); + HUF_FLUSHBITS_1(&bitC); + case 1 : HUF_encodeSymbol(&bitC, ip[n+ 0], CTable); + HUF_FLUSHBITS(&bitC); + case 0 : + default: ; + } + + for (; n>0; n-=4) { /* note : n&3==0 at this stage */ + HUF_encodeSymbol(&bitC, ip[n- 1], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 2], CTable); + HUF_FLUSHBITS_2(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 3], CTable); + HUF_FLUSHBITS_1(&bitC); + HUF_encodeSymbol(&bitC, ip[n- 4], CTable); + HUF_FLUSHBITS(&bitC); + } + + return BIT_closeCStream(&bitC); +} + + +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +{ + size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ + const BYTE* ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + if (dstSize < 6 + 1 + 1 + 1 + 8) return 0; /* minimum space to compress successfully */ + if (srcSize < 12) return 0; /* no saving possible : too small input */ + op += 6; /* jumpTable */ + + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart+2, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, segmentSize, CTable) ); + if (cSize==0) return 0; + MEM_writeLE16(ostart+4, (U16)cSize); + op += cSize; + } + + ip += segmentSize; + { CHECK_V_F(cSize, HUF_compress1X_usingCTable(op, oend-op, ip, iend-ip, CTable) ); + if (cSize==0) return 0; + op += cSize; + } + + return op-ostart; +} + + +/* `workSpace` must a table of at least 1024 unsigned */ +static size_t HUF_compress_internal ( + void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + unsigned singleStream, + void* workSpace, size_t wkspSize) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstSize; + BYTE* op = ostart; + + union { + U32 count[HUF_SYMBOLVALUE_MAX+1]; + HUF_CElt CTable[HUF_SYMBOLVALUE_MAX+1]; + } table; /* `count` can overlap with `CTable`; saves 1 KB */ + + /* checks & inits */ + if (wkspSize < sizeof(huffNodeTable)) return ERROR(GENERIC); + if (!srcSize) return 0; /* Uncompressed (note : 1 means rle, so first byte must be correct) */ + if (!dstSize) return 0; /* cannot fit within dst budget */ + if (srcSize > HUF_BLOCKSIZE_MAX) return ERROR(srcSize_wrong); /* current block size limit */ + if (huffLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + if (!maxSymbolValue) maxSymbolValue = HUF_SYMBOLVALUE_MAX; + if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; + + /* Scan input and build symbol stats */ + { CHECK_V_F(largest, FSE_count_wksp (table.count, &maxSymbolValue, (const BYTE*)src, srcSize, (U32*)workSpace) ); + if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ + if (largest <= (srcSize >> 7)+1) return 0; /* Fast heuristic : not compressible enough */ + } + + /* Build Huffman Tree */ + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + { CHECK_V_F(maxBits, HUF_buildCTable_wksp (table.CTable, table.count, maxSymbolValue, huffLog, workSpace, wkspSize) ); + huffLog = (U32)maxBits; + } + + /* Write table description header */ + { CHECK_V_F(hSize, HUF_writeCTable (op, dstSize, table.CTable, maxSymbolValue, huffLog) ); + if (hSize + 12 >= srcSize) return 0; /* not useful to try compression */ + op += hSize; + } + + /* Compress */ + { size_t const cSize = (singleStream) ? + HUF_compress1X_usingCTable(op, oend - op, src, srcSize, table.CTable) : /* single segment */ + HUF_compress4X_usingCTable(op, oend - op, src, srcSize, table.CTable); + if (HUF_isError(cSize)) return cSize; + if (cSize==0) return 0; /* uncompressible */ + op += cSize; + } + + /* check compressibility */ + if ((size_t)(op-ostart) >= srcSize-1) + return 0; + + return op-ostart; +} + + +size_t HUF_compress1X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 1 /* single stream */, workSpace, wkspSize); +} + +size_t HUF_compress1X (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[1024]; + return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress4X_wksp (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog, + void* workSpace, size_t wkspSize) +{ + return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, 0 /* 4 streams */, workSpace, wkspSize); +} + +size_t HUF_compress2 (void* dst, size_t dstSize, + const void* src, size_t srcSize, + unsigned maxSymbolValue, unsigned huffLog) +{ + unsigned workSpace[1024]; + return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); +} + +size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) +{ + return HUF_compress2(dst, maxDstSize, src, (U32)srcSize, 255, HUF_TABLELOG_DEFAULT); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/zstd_compress.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/zstd_compress.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/zstd_compress.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/zstd_compress.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,3291 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/*-************************************* +* Dependencies +***************************************/ +#include /* memset */ +#include "mem.h" +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH_reset, update, digest */ +#define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" /* includes zstd.h */ + + +/*-************************************* +* Constants +***************************************/ +static const U32 g_searchStrength = 8; /* control skip over incompressible data */ +#define HASH_READ_SIZE 8 +typedef enum { ZSTDcs_created=0, ZSTDcs_init, ZSTDcs_ongoing, ZSTDcs_ending } ZSTD_compressionStage_e; + + +/*-************************************* +* Helper functions +***************************************/ +#define ZSTD_STATIC_ASSERT(c) { enum { ZSTD_static_assert = 1/(int)(!!(c)) }; } +size_t ZSTD_compressBound(size_t srcSize) { return FSE_compressBound(srcSize) + 12; } + + +/*-************************************* +* Sequence storage +***************************************/ +static void ZSTD_resetSeqStore(seqStore_t* ssPtr) +{ + ssPtr->lit = ssPtr->litStart; + ssPtr->sequences = ssPtr->sequencesStart; + ssPtr->longLengthID = 0; +} + + +/*-************************************* +* Context memory management +***************************************/ +struct ZSTD_CCtx_s +{ + const BYTE* nextSrc; /* next block here to continue on current prefix */ + const BYTE* base; /* All regular indexes relative to this position */ + const BYTE* dictBase; /* extDict indexes relative to this position */ + U32 dictLimit; /* below that point, need extDict */ + U32 lowLimit; /* below that point, no more data */ + U32 nextToUpdate; /* index from which to continue dictionary update */ + U32 nextToUpdate3; /* index from which to continue dictionary update */ + U32 hashLog3; /* dispatch table : larger == faster, more memory */ + U32 loadedDictEnd; + ZSTD_compressionStage_e stage; + U32 rep[ZSTD_REP_NUM]; + U32 savedRep[ZSTD_REP_NUM]; + U32 dictID; + ZSTD_parameters params; + void* workSpace; + size_t workSpaceSize; + size_t blockSize; + U64 frameContentSize; + XXH64_state_t xxhState; + ZSTD_customMem customMem; + + seqStore_t seqStore; /* sequences storage ptrs */ + U32* hashTable; + U32* hashTable3; + U32* chainTable; + HUF_CElt* hufTable; + U32 flagStaticTables; + FSE_CTable offcodeCTable [FSE_CTABLE_SIZE_U32(OffFSELog, MaxOff)]; + FSE_CTable matchlengthCTable[FSE_CTABLE_SIZE_U32(MLFSELog, MaxML)]; + FSE_CTable litlengthCTable [FSE_CTABLE_SIZE_U32(LLFSELog, MaxLL)]; + unsigned tmpCounters[1024]; +}; + +ZSTD_CCtx* ZSTD_createCCtx(void) +{ + return ZSTD_createCCtx_advanced(defaultCustomMem); +} + +ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_CCtx* cctx; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + cctx = (ZSTD_CCtx*) ZSTD_malloc(sizeof(ZSTD_CCtx), customMem); + if (!cctx) return NULL; + memset(cctx, 0, sizeof(ZSTD_CCtx)); + memcpy(&(cctx->customMem), &customMem, sizeof(customMem)); + return cctx; +} + +size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support free on NULL */ + ZSTD_free(cctx->workSpace, cctx->customMem); + ZSTD_free(cctx, cctx->customMem); + return 0; /* reserved as a potential error code in the future */ +} + +size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx) +{ + if (cctx==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*cctx) + cctx->workSpaceSize; +} + +const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx) /* hidden interface */ +{ + return &(ctx->seqStore); +} + +static ZSTD_parameters ZSTD_getParamsFromCCtx(const ZSTD_CCtx* cctx) +{ + return cctx->params; +} + + +/** ZSTD_checkParams() : + ensure param values remain within authorized range. + @return : 0, or an error code if one value is beyond authorized range */ +size_t ZSTD_checkCParams(ZSTD_compressionParameters cParams) +{ +# define CLAMPCHECK(val,min,max) { if ((valmax)) return ERROR(compressionParameter_unsupported); } + CLAMPCHECK(cParams.windowLog, ZSTD_WINDOWLOG_MIN, ZSTD_WINDOWLOG_MAX); + CLAMPCHECK(cParams.chainLog, ZSTD_CHAINLOG_MIN, ZSTD_CHAINLOG_MAX); + CLAMPCHECK(cParams.hashLog, ZSTD_HASHLOG_MIN, ZSTD_HASHLOG_MAX); + CLAMPCHECK(cParams.searchLog, ZSTD_SEARCHLOG_MIN, ZSTD_SEARCHLOG_MAX); + { U32 const searchLengthMin = ((cParams.strategy == ZSTD_fast) | (cParams.strategy == ZSTD_greedy)) ? ZSTD_SEARCHLENGTH_MIN+1 : ZSTD_SEARCHLENGTH_MIN; + U32 const searchLengthMax = (cParams.strategy == ZSTD_fast) ? ZSTD_SEARCHLENGTH_MAX : ZSTD_SEARCHLENGTH_MAX-1; + CLAMPCHECK(cParams.searchLength, searchLengthMin, searchLengthMax); } + CLAMPCHECK(cParams.targetLength, ZSTD_TARGETLENGTH_MIN, ZSTD_TARGETLENGTH_MAX); + if ((U32)(cParams.strategy) > (U32)ZSTD_btopt2) return ERROR(compressionParameter_unsupported); + return 0; +} + + +/** ZSTD_cycleLog() : + * condition for correct operation : hashLog > 1 */ +static U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat) +{ + U32 const btScale = ((U32)strat >= (U32)ZSTD_btlazy2); + return hashLog - btScale; +} + +/** ZSTD_adjustCParams() : + optimize `cPar` for a given input (`srcSize` and `dictSize`). + mostly downsizing to reduce memory consumption and initialization. + Both `srcSize` and `dictSize` are optional (use 0 if unknown), + but if both are 0, no optimization can be done. + Note : cPar is considered validated at this stage. Use ZSTD_checkParams() to ensure that. */ +ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize) +{ + if (srcSize+dictSize == 0) return cPar; /* no size information available : no adjustment */ + + /* resize params, to use less memory when necessary */ + { U32 const minSrcSize = (srcSize==0) ? 500 : 0; + U64 const rSize = srcSize + dictSize + minSrcSize; + if (rSize < ((U64)1< srcLog) cPar.windowLog = srcLog; + } } + if (cPar.hashLog > cPar.windowLog) cPar.hashLog = cPar.windowLog; + { U32 const cycleLog = ZSTD_cycleLog(cPar.chainLog, cPar.strategy); + if (cycleLog > cPar.windowLog) cPar.chainLog -= (cycleLog - cPar.windowLog); + } + + if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* required for frame header */ + + return cPar; +} + + +size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams) +{ + size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << cParams.windowLog); + U32 const divider = (cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + + size_t const chainSize = (cParams.strategy == ZSTD_fast) ? 0 : (1 << cParams.chainLog); + size_t const hSize = ((size_t)1) << cParams.hashLog; + U32 const hashLog3 = (cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + + size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<nextSrc - cctx->base); + cctx->params = params; + cctx->frameContentSize = frameContentSize; + cctx->lowLimit = end; + cctx->dictLimit = end; + cctx->nextToUpdate = end+1; + cctx->stage = ZSTDcs_init; + cctx->dictID = 0; + cctx->loadedDictEnd = 0; + { int i; for (i=0; irep[i] = repStartValue[i]; } + cctx->seqStore.litLengthSum = 0; /* force reset of btopt stats */ + XXH64_reset(&cctx->xxhState, 0); + return 0; +} + +typedef enum { ZSTDcrp_continue, ZSTDcrp_noMemset, ZSTDcrp_fullReset } ZSTD_compResetPolicy_e; + +/*! ZSTD_resetCCtx_advanced() : + note : 'params' must be validated */ +static size_t ZSTD_resetCCtx_advanced (ZSTD_CCtx* zc, + ZSTD_parameters params, U64 frameContentSize, + ZSTD_compResetPolicy_e const crp) +{ + if (crp == ZSTDcrp_continue) + if (ZSTD_equivalentParams(params, zc->params)) + return ZSTD_continueCCtx(zc, params, frameContentSize); + + { size_t const blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, (size_t)1 << params.cParams.windowLog); + U32 const divider = (params.cParams.searchLength==3) ? 3 : 4; + size_t const maxNbSeq = blockSize / divider; + size_t const tokenSpace = blockSize + 11*maxNbSeq; + size_t const chainSize = (params.cParams.strategy == ZSTD_fast) ? 0 : (1 << params.cParams.chainLog); + size_t const hSize = ((size_t)1) << params.cParams.hashLog; + U32 const hashLog3 = (params.cParams.searchLength>3) ? 0 : MIN(ZSTD_HASHLOG3_MAX, params.cParams.windowLog); + size_t const h3Size = ((size_t)1) << hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + void* ptr; + + /* Check if workSpace is large enough, alloc a new one if needed */ + { size_t const optSpace = ((MaxML+1) + (MaxLL+1) + (MaxOff+1) + (1<workSpaceSize < neededSpace) { + ZSTD_free(zc->workSpace, zc->customMem); + zc->workSpace = ZSTD_malloc(neededSpace, zc->customMem); + if (zc->workSpace == NULL) return ERROR(memory_allocation); + zc->workSpaceSize = neededSpace; + } } + + if (crp!=ZSTDcrp_noMemset) memset(zc->workSpace, 0, tableSpace); /* reset tables only */ + XXH64_reset(&zc->xxhState, 0); + zc->hashLog3 = hashLog3; + zc->hashTable = (U32*)(zc->workSpace); + zc->chainTable = zc->hashTable + hSize; + zc->hashTable3 = zc->chainTable + chainSize; + ptr = zc->hashTable3 + h3Size; + zc->hufTable = (HUF_CElt*)ptr; + zc->flagStaticTables = 0; + ptr = ((U32*)ptr) + 256; /* note : HUF_CElt* is incomplete type, size is simulated using U32 */ + + zc->nextToUpdate = 1; + zc->nextSrc = NULL; + zc->base = NULL; + zc->dictBase = NULL; + zc->dictLimit = 0; + zc->lowLimit = 0; + zc->params = params; + zc->blockSize = blockSize; + zc->frameContentSize = frameContentSize; + { int i; for (i=0; irep[i] = repStartValue[i]; } + + if ((params.cParams.strategy == ZSTD_btopt) || (params.cParams.strategy == ZSTD_btopt2)) { + zc->seqStore.litFreq = (U32*)ptr; + zc->seqStore.litLengthFreq = zc->seqStore.litFreq + (1<seqStore.matchLengthFreq = zc->seqStore.litLengthFreq + (MaxLL+1); + zc->seqStore.offCodeFreq = zc->seqStore.matchLengthFreq + (MaxML+1); + ptr = zc->seqStore.offCodeFreq + (MaxOff+1); + zc->seqStore.matchTable = (ZSTD_match_t*)ptr; + ptr = zc->seqStore.matchTable + ZSTD_OPT_NUM+1; + zc->seqStore.priceTable = (ZSTD_optimal_t*)ptr; + ptr = zc->seqStore.priceTable + ZSTD_OPT_NUM+1; + zc->seqStore.litLengthSum = 0; + } + zc->seqStore.sequencesStart = (seqDef*)ptr; + ptr = zc->seqStore.sequencesStart + maxNbSeq; + zc->seqStore.llCode = (BYTE*) ptr; + zc->seqStore.mlCode = zc->seqStore.llCode + maxNbSeq; + zc->seqStore.ofCode = zc->seqStore.mlCode + maxNbSeq; + zc->seqStore.litStart = zc->seqStore.ofCode + maxNbSeq; + + zc->stage = ZSTDcs_init; + zc->dictID = 0; + zc->loadedDictEnd = 0; + + return 0; + } +} + + +/*! ZSTD_copyCCtx() : +* Duplicate an existing context `srcCCtx` into another one `dstCCtx`. +* Only works during stage ZSTDcs_init (i.e. after creation, but before first call to ZSTD_compressContinue()). +* @return : 0, or an error code */ +size_t ZSTD_copyCCtx(ZSTD_CCtx* dstCCtx, const ZSTD_CCtx* srcCCtx, unsigned long long pledgedSrcSize) +{ + if (srcCCtx->stage!=ZSTDcs_init) return ERROR(stage_wrong); + + memcpy(&dstCCtx->customMem, &srcCCtx->customMem, sizeof(ZSTD_customMem)); + ZSTD_resetCCtx_advanced(dstCCtx, srcCCtx->params, pledgedSrcSize, ZSTDcrp_noMemset); + + /* copy tables */ + { size_t const chainSize = (srcCCtx->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << srcCCtx->params.cParams.chainLog); + size_t const hSize = ((size_t)1) << srcCCtx->params.cParams.hashLog; + size_t const h3Size = (size_t)1 << srcCCtx->hashLog3; + size_t const tableSpace = (chainSize + hSize + h3Size) * sizeof(U32); + memcpy(dstCCtx->workSpace, srcCCtx->workSpace, tableSpace); + } + + /* copy dictionary offsets */ + dstCCtx->nextToUpdate = srcCCtx->nextToUpdate; + dstCCtx->nextToUpdate3= srcCCtx->nextToUpdate3; + dstCCtx->nextSrc = srcCCtx->nextSrc; + dstCCtx->base = srcCCtx->base; + dstCCtx->dictBase = srcCCtx->dictBase; + dstCCtx->dictLimit = srcCCtx->dictLimit; + dstCCtx->lowLimit = srcCCtx->lowLimit; + dstCCtx->loadedDictEnd= srcCCtx->loadedDictEnd; + dstCCtx->dictID = srcCCtx->dictID; + + /* copy entropy tables */ + dstCCtx->flagStaticTables = srcCCtx->flagStaticTables; + if (srcCCtx->flagStaticTables) { + memcpy(dstCCtx->hufTable, srcCCtx->hufTable, 256*4); + memcpy(dstCCtx->litlengthCTable, srcCCtx->litlengthCTable, sizeof(dstCCtx->litlengthCTable)); + memcpy(dstCCtx->matchlengthCTable, srcCCtx->matchlengthCTable, sizeof(dstCCtx->matchlengthCTable)); + memcpy(dstCCtx->offcodeCTable, srcCCtx->offcodeCTable, sizeof(dstCCtx->offcodeCTable)); + } + + return 0; +} + + +/*! ZSTD_reduceTable() : +* reduce table indexes by `reducerValue` */ +static void ZSTD_reduceTable (U32* const table, U32 const size, U32 const reducerValue) +{ + U32 u; + for (u=0 ; u < size ; u++) { + if (table[u] < reducerValue) table[u] = 0; + else table[u] -= reducerValue; + } +} + +/*! ZSTD_reduceIndex() : +* rescale all indexes to avoid future overflow (indexes are U32) */ +static void ZSTD_reduceIndex (ZSTD_CCtx* zc, const U32 reducerValue) +{ + { U32 const hSize = 1 << zc->params.cParams.hashLog; + ZSTD_reduceTable(zc->hashTable, hSize, reducerValue); } + + { U32 const chainSize = (zc->params.cParams.strategy == ZSTD_fast) ? 0 : (1 << zc->params.cParams.chainLog); + ZSTD_reduceTable(zc->chainTable, chainSize, reducerValue); } + + { U32 const h3Size = (zc->hashLog3) ? 1 << zc->hashLog3 : 0; + ZSTD_reduceTable(zc->hashTable3, h3Size, reducerValue); } +} + + +/*-******************************************************* +* Block entropic compression +*********************************************************/ + +/* See doc/zstd_compression_format.md for detailed format description */ + +size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy((BYTE*)dst + ZSTD_blockHeaderSize, src, srcSize); + MEM_writeLE24(dst, (U32)(srcSize << 2) + (U32)bt_raw); + return ZSTD_blockHeaderSize+srcSize; +} + + +static size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + if (srcSize + flSize > dstCapacity) return ERROR(dstSize_tooSmall); + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_basic + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_basic + (1<<2) + (srcSize<<4))); + break; + default: /*note : should not be necessary : flSize is within {1,2,3} */ + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_basic + (3<<2) + (srcSize<<4))); + break; + } + + memcpy(ostart + flSize, src, srcSize); + return srcSize + flSize; +} + +static size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + BYTE* const ostart = (BYTE* const)dst; + U32 const flSize = 1 + (srcSize>31) + (srcSize>4095); + + (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + + switch(flSize) + { + case 1: /* 2 - 1 - 5 */ + ostart[0] = (BYTE)((U32)set_rle + (srcSize<<3)); + break; + case 2: /* 2 - 2 - 12 */ + MEM_writeLE16(ostart, (U16)((U32)set_rle + (1<<2) + (srcSize<<4))); + break; + default: /*note : should not be necessary : flSize is necessarily within {1,2,3} */ + case 3: /* 2 - 2 - 20 */ + MEM_writeLE32(ostart, (U32)((U32)set_rle + (3<<2) + (srcSize<<4))); + break; + } + + ostart[flSize] = *(const BYTE*)src; + return flSize+1; +} + + +static size_t ZSTD_minGain(size_t srcSize) { return (srcSize >> 6) + 2; } + +static size_t ZSTD_compressLiterals (ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t const minGain = ZSTD_minGain(srcSize); + size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); + BYTE* const ostart = (BYTE*)dst; + U32 singleStream = srcSize < 256; + symbolEncodingType_e hType = set_compressed; + size_t cLitSize; + + + /* small ? don't even attempt compression (speed opt) */ +# define LITERAL_NOENTROPY 63 + { size_t const minLitSize = zc->flagStaticTables ? 6 : LITERAL_NOENTROPY; + if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } + + if (dstCapacity < lhSize+1) return ERROR(dstSize_tooSmall); /* not enough space for compression */ + if (zc->flagStaticTables && (lhSize==3)) { + hType = set_repeat; + singleStream = 1; + cLitSize = HUF_compress1X_usingCTable(ostart+lhSize, dstCapacity-lhSize, src, srcSize, zc->hufTable); + } else { + cLitSize = singleStream ? HUF_compress1X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters)) + : HUF_compress4X_wksp(ostart+lhSize, dstCapacity-lhSize, src, srcSize, 255, 11, zc->tmpCounters, sizeof(zc->tmpCounters)); + } + + if ((cLitSize==0) | (cLitSize >= srcSize - minGain)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + if (cLitSize==1) + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + + /* Build header */ + switch(lhSize) + { + case 3: /* 2 - 2 - 10 - 10 */ + { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + MEM_writeLE24(ostart, lhc); + break; + } + case 4: /* 2 - 2 - 14 - 14 */ + { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); + MEM_writeLE32(ostart, lhc); + break; + } + default: /* should not be necessary, lhSize is only {3,4,5} */ + case 5: /* 2 - 2 - 18 - 18 */ + { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); + MEM_writeLE32(ostart, lhc); + ostart[4] = (BYTE)(cLitSize >> 10); + break; + } + } + return lhSize+cLitSize; +} + +static const BYTE LL_Code[64] = { 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 16, 17, 17, 18, 18, 19, 19, + 20, 20, 20, 20, 21, 21, 21, 21, + 22, 22, 22, 22, 22, 22, 22, 22, + 23, 23, 23, 23, 23, 23, 23, 23, + 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24 }; + +static const BYTE ML_Code[128] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, + 32, 32, 33, 33, 34, 34, 35, 35, 36, 36, 36, 36, 37, 37, 37, 37, + 38, 38, 38, 38, 38, 38, 38, 38, 39, 39, 39, 39, 39, 39, 39, 39, + 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, + 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, + 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42, 42 }; + + +void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +{ + BYTE const LL_deltaCode = 19; + BYTE const ML_deltaCode = 36; + const seqDef* const sequences = seqStorePtr->sequencesStart; + BYTE* const llCodeTable = seqStorePtr->llCode; + BYTE* const ofCodeTable = seqStorePtr->ofCode; + BYTE* const mlCodeTable = seqStorePtr->mlCode; + U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + U32 u; + for (u=0; u 63) ? (BYTE)ZSTD_highbit32(llv) + LL_deltaCode : LL_Code[llv]; + ofCodeTable[u] = (BYTE)ZSTD_highbit32(sequences[u].offset); + mlCodeTable[u] = (mlv>127) ? (BYTE)ZSTD_highbit32(mlv) + ML_deltaCode : ML_Code[mlv]; + } + if (seqStorePtr->longLengthID==1) + llCodeTable[seqStorePtr->longLengthPos] = MaxLL; + if (seqStorePtr->longLengthID==2) + mlCodeTable[seqStorePtr->longLengthPos] = MaxML; +} + + +size_t ZSTD_compressSequences(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + size_t srcSize) +{ + const seqStore_t* seqStorePtr = &(zc->seqStore); + U32 count[MaxSeq+1]; + S16 norm[MaxSeq+1]; + FSE_CTable* CTable_LitLength = zc->litlengthCTable; + FSE_CTable* CTable_OffsetBits = zc->offcodeCTable; + FSE_CTable* CTable_MatchLength = zc->matchlengthCTable; + U32 LLtype, Offtype, MLtype; /* compressed, raw or rle */ + const seqDef* const sequences = seqStorePtr->sequencesStart; + const BYTE* const ofCodeTable = seqStorePtr->ofCode; + const BYTE* const llCodeTable = seqStorePtr->llCode; + const BYTE* const mlCodeTable = seqStorePtr->mlCode; + BYTE* const ostart = (BYTE*)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + BYTE* seqHead; + BYTE scratchBuffer[1<litStart; + size_t const litSize = seqStorePtr->lit - literals; + size_t const cSize = ZSTD_compressLiterals(zc, op, dstCapacity, literals, litSize); + if (ZSTD_isError(cSize)) return cSize; + op += cSize; + } + + /* Sequences Header */ + if ((oend-op) < 3 /*max nbSeq Size*/ + 1 /*seqHead */) return ERROR(dstSize_tooSmall); + if (nbSeq < 0x7F) *op++ = (BYTE)nbSeq; + else if (nbSeq < LONGNBSEQ) op[0] = (BYTE)((nbSeq>>8) + 0x80), op[1] = (BYTE)nbSeq, op+=2; + else op[0]=0xFF, MEM_writeLE16(op+1, (U16)(nbSeq - LONGNBSEQ)), op+=3; + if (nbSeq==0) goto _check_compressibility; + + /* seqHead : flags for FSE encoding type */ + seqHead = op++; + +#define MIN_SEQ_FOR_DYNAMIC_FSE 64 +#define MAX_SEQ_FOR_STATIC_FSE 1000 + + /* convert length/distances into codes */ + ZSTD_seqToCodes(seqStorePtr); + + /* CTable for Literal Lengths */ + { U32 max = MaxLL; + size_t const mostFrequent = FSE_countFast_wksp(count, &max, llCodeTable, nbSeq, zc->tmpCounters); + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { + *op++ = llCodeTable[0]; + FSE_buildCTable_rle(CTable_LitLength, (BYTE)max); + LLtype = set_rle; + } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + LLtype = set_repeat; + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (LL_defaultNormLog-1)))) { + FSE_buildCTable_wksp(CTable_LitLength, LL_defaultNorm, MaxLL, LL_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + LLtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(LLFSELog, nbSeq, max); + if (count[llCodeTable[nbSeq-1]]>1) { count[llCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable_wksp(CTable_LitLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + LLtype = set_compressed; + } } + + /* CTable for Offsets */ + { U32 max = MaxOff; + size_t const mostFrequent = FSE_countFast_wksp(count, &max, ofCodeTable, nbSeq, zc->tmpCounters); + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { + *op++ = ofCodeTable[0]; + FSE_buildCTable_rle(CTable_OffsetBits, (BYTE)max); + Offtype = set_rle; + } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + Offtype = set_repeat; + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (OF_defaultNormLog-1)))) { + FSE_buildCTable_wksp(CTable_OffsetBits, OF_defaultNorm, MaxOff, OF_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + Offtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(OffFSELog, nbSeq, max); + if (count[ofCodeTable[nbSeq-1]]>1) { count[ofCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable_wksp(CTable_OffsetBits, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + Offtype = set_compressed; + } } + + /* CTable for MatchLengths */ + { U32 max = MaxML; + size_t const mostFrequent = FSE_countFast_wksp(count, &max, mlCodeTable, nbSeq, zc->tmpCounters); + if ((mostFrequent == nbSeq) && (nbSeq > 2)) { + *op++ = *mlCodeTable; + FSE_buildCTable_rle(CTable_MatchLength, (BYTE)max); + MLtype = set_rle; + } else if ((zc->flagStaticTables) && (nbSeq < MAX_SEQ_FOR_STATIC_FSE)) { + MLtype = set_repeat; + } else if ((nbSeq < MIN_SEQ_FOR_DYNAMIC_FSE) || (mostFrequent < (nbSeq >> (ML_defaultNormLog-1)))) { + FSE_buildCTable_wksp(CTable_MatchLength, ML_defaultNorm, MaxML, ML_defaultNormLog, scratchBuffer, sizeof(scratchBuffer)); + MLtype = set_basic; + } else { + size_t nbSeq_1 = nbSeq; + const U32 tableLog = FSE_optimalTableLog(MLFSELog, nbSeq, max); + if (count[mlCodeTable[nbSeq-1]]>1) { count[mlCodeTable[nbSeq-1]]--; nbSeq_1--; } + FSE_normalizeCount(norm, tableLog, count, nbSeq_1, max); + { size_t const NCountSize = FSE_writeNCount(op, oend-op, norm, max, tableLog); /* overflow protected */ + if (FSE_isError(NCountSize)) return ERROR(GENERIC); + op += NCountSize; } + FSE_buildCTable_wksp(CTable_MatchLength, norm, max, tableLog, scratchBuffer, sizeof(scratchBuffer)); + MLtype = set_compressed; + } } + + *seqHead = (BYTE)((LLtype<<6) + (Offtype<<4) + (MLtype<<2)); + zc->flagStaticTables = 0; + + /* Encoding Sequences */ + { BIT_CStream_t blockStream; + FSE_CState_t stateMatchLength; + FSE_CState_t stateOffsetBits; + FSE_CState_t stateLitLength; + + CHECK_E(BIT_initCStream(&blockStream, op, oend-op), dstSize_tooSmall); /* not enough space remaining */ + + /* first symbols */ + FSE_initCState2(&stateMatchLength, CTable_MatchLength, mlCodeTable[nbSeq-1]); + FSE_initCState2(&stateOffsetBits, CTable_OffsetBits, ofCodeTable[nbSeq-1]); + FSE_initCState2(&stateLitLength, CTable_LitLength, llCodeTable[nbSeq-1]); + BIT_addBits(&blockStream, sequences[nbSeq-1].litLength, LL_bits[llCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].matchLength, ML_bits[mlCodeTable[nbSeq-1]]); + if (MEM_32bits()) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[nbSeq-1].offset, ofCodeTable[nbSeq-1]); + BIT_flushBits(&blockStream); + + { size_t n; + for (n=nbSeq-2 ; n= 64-7-(LLFSELog+MLFSELog+OffFSELog))) + BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].litLength, llBits); + if (MEM_32bits() && ((llBits+mlBits)>24)) BIT_flushBits(&blockStream); + BIT_addBits(&blockStream, sequences[n].matchLength, mlBits); + if (MEM_32bits()) BIT_flushBits(&blockStream); /* (7)*/ + BIT_addBits(&blockStream, sequences[n].offset, ofBits); /* 31 */ + BIT_flushBits(&blockStream); /* (7)*/ + } } + + FSE_flushCState(&blockStream, &stateMatchLength); + FSE_flushCState(&blockStream, &stateOffsetBits); + FSE_flushCState(&blockStream, &stateLitLength); + + { size_t const streamSize = BIT_closeCStream(&blockStream); + if (streamSize==0) return ERROR(dstSize_tooSmall); /* not enough space */ + op += streamSize; + } } + + /* check compressibility */ +_check_compressibility: + { size_t const minGain = ZSTD_minGain(srcSize); + size_t const maxCSize = srcSize - minGain; + if ((size_t)(op-ostart) >= maxCSize) return 0; } + + /* confirm repcodes */ + { int i; for (i=0; irep[i] = zc->savedRep[i]; } + + return op - ostart; +} + + +/*! ZSTD_storeSeq() : + Store a sequence (literal length, literals, offset code and match length code) into seqStore_t. + `offsetCode` : distance to match, or 0 == repCode. + `matchCode` : matchLength - MINMATCH +*/ +MEM_STATIC void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const void* literals, U32 offsetCode, size_t matchCode) +{ +#if 0 /* for debug */ + static const BYTE* g_start = NULL; + const U32 pos = (U32)((const BYTE*)literals - g_start); + if (g_start==NULL) g_start = (const BYTE*)literals; + //if ((pos > 1) && (pos < 50000)) + printf("Cpos %6u :%5u literals & match %3u bytes at distance %6u \n", + pos, (U32)litLength, (U32)matchCode+MINMATCH, (U32)offsetCode); +#endif + /* copy Literals */ + ZSTD_wildcopy(seqStorePtr->lit, literals, litLength); + seqStorePtr->lit += litLength; + + /* literal Length */ + if (litLength>0xFFFF) { seqStorePtr->longLengthID = 1; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + seqStorePtr->sequences[0].litLength = (U16)litLength; + + /* match offset */ + seqStorePtr->sequences[0].offset = offsetCode + 1; + + /* match Length */ + if (matchCode>0xFFFF) { seqStorePtr->longLengthID = 2; seqStorePtr->longLengthPos = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); } + seqStorePtr->sequences[0].matchLength = (U16)matchCode; + + seqStorePtr->sequences++; +} + + +/*-************************************* +* Match length counter +***************************************/ +static unsigned ZSTD_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +static size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) +{ + const BYTE* const pStart = pIn; + const BYTE* const pInLoopLimit = pInLimit - (sizeof(size_t)-1); + + while (pIn < pInLoopLimit) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { pIn+=sizeof(size_t); pMatch+=sizeof(size_t); continue; } + pIn += ZSTD_NbCommonBytes(diff); + return (size_t)(pIn - pStart); + } + if (MEM_64bits()) if ((pIn<(pInLimit-3)) && (MEM_read32(pMatch) == MEM_read32(pIn))) { pIn+=4; pMatch+=4; } + if ((pIn<(pInLimit-1)) && (MEM_read16(pMatch) == MEM_read16(pIn))) { pIn+=2; pMatch+=2; } + if ((pIn> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ + +static const U32 prime4bytes = 2654435761U; +static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } + +static const U64 prime5bytes = 889523592379ULL; +static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } + +static const U64 prime6bytes = 227718039650203ULL; +static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } + +static const U64 prime7bytes = 58295818150454627ULL; +static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } + +static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; +static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } + +static size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) +{ + switch(mls) + { + default: + case 4: return ZSTD_hash4Ptr(p, hBits); + case 5: return ZSTD_hash5Ptr(p, hBits); + case 6: return ZSTD_hash6Ptr(p, hBits); + case 7: return ZSTD_hash7Ptr(p, hBits); + case 8: return ZSTD_hash8Ptr(p, hBits); + } +} + + +/*-************************************* +* Fast Scan +***************************************/ +static void ZSTD_fillHashTable (ZSTD_CCtx* zc, const void* end, const U32 mls) +{ + U32* const hashTable = zc->hashTable; + U32 const hBits = zc->params.cParams.hashLog; + const BYTE* const base = zc->base; + const BYTE* ip = base + zc->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_fast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashTable = cctx->hashTable; + U32 const hBits = cctx->params.cParams.hashLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h = ZSTD_hashPtr(ip, hBits, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndex = hashTable[h]; + const BYTE* match = base + matchIndex; + hashTable[h] = current; /* update hash table */ + + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + U32 offset; + if ( (matchIndex <= lowestIndex) || (MEM_read32(match) != MEM_read32(ip)) ) { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; + cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_fast(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + const U32 mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_fast_generic(ctx, src, srcSize, 7); return; + } +} + + +static void ZSTD_compressBlock_fast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* hashTable = ctx->hashTable; + const U32 hBits = ctx->params.cParams.hashLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t h = ZSTD_hashPtr(ip, hBits, mls); + const U32 matchIndex = hashTable[h]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashTable[h] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repMatchEnd, lowPrefixPtr) + EQUAL_READ32; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ( (matchIndex < lowestIndex) || + (MEM_read32(match) != MEM_read32(ip)) ) { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + { const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iend, matchEnd, lowPrefixPtr) + EQUAL_READ32; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + offset = current - matchIndex; + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashTable[ZSTD_hashPtr(base+current+2, hBits, mls)] = current+2; + hashTable[ZSTD_hashPtr(ip-2, hBits, mls)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashTable[ZSTD_hashPtr(ip, hBits, mls)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_fast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + U32 const mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_fast_extDict_generic(ctx, src, srcSize, 7); return; + } +} + + +/*-************************************* +* Double Fast +***************************************/ +static void ZSTD_fillDoubleHashTable (ZSTD_CCtx* cctx, const void* end, const U32 mls) +{ + U32* const hashLarge = cctx->hashTable; + U32 const hBitsL = cctx->params.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + U32 const hBitsS = cctx->params.cParams.chainLog; + const BYTE* const base = cctx->base; + const BYTE* ip = base + cctx->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const size_t fastHashFillStep = 3; + + while(ip <= iend) { + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip - base); + hashLarge[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip - base); + ip += fastHashFillStep; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_doubleFast_generic(ZSTD_CCtx* cctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = cctx->hashTable; + const U32 hBitsL = cctx->params.cParams.hashLog; + U32* const hashSmall = cctx->chainTable; + const U32 hBitsS = cctx->params.cParams.chainLog; + seqStore_t* seqStorePtr = &(cctx->seqStore); + const BYTE* const base = cctx->base; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = cctx->dictLimit; + const BYTE* const lowest = base + lowestIndex; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - HASH_READ_SIZE; + U32 offset_1=cctx->rep[0], offset_2=cctx->rep[1]; + U32 offsetSaved = 0; + + /* init */ + ip += (ip==lowest); + { U32 const maxRep = (U32)(ip-lowest); + if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + } + + /* Main Search Loop */ + while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + size_t mLength; + size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); + size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); + U32 const current = (U32)(ip-base); + U32 const matchIndexL = hashLong[h2]; + U32 const matchIndexS = hashSmall[h]; + const BYTE* matchLong = base + matchIndexL; + const BYTE* match = base + matchIndexS; + hashLong[h2] = hashSmall[h] = current; /* update hash tables */ + + if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { /* note : by construction, offset_1 <= current */ + mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + U32 offset; + if ( (matchIndexL > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip)) ) { + mLength = ZSTD_count(ip+8, matchLong+8, iend) + 8; + offset = (U32)(ip-matchLong); + while (((ip>anchor) & (matchLong>lowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + } else if ( (matchIndexS > lowestIndex) && (MEM_read32(match) == MEM_read32(ip)) ) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* match3 = base + matchIndex3; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + mLength = ZSTD_count(ip+9, match3+8, iend) + 8; + ip++; + offset = (U32)(ip-match3); + while (((ip>anchor) & (match3>lowest)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + mLength = ZSTD_count(ip+4, match+4, iend) + 4; + offset = (U32)(ip-match); + while (((ip>anchor) & (match>lowest)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } + + offset_2 = offset_1; + offset_1 = offset; + + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + } + + /* match found */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; /* here because current+2 could be > iend-8 */ + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ( (offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + size_t const rLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; + { U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; } /* swap offset_2 <=> offset_1 */ + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, rLength-MINMATCH); + ip += rLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } } + + /* save reps for next block */ + cctx->savedRep[0] = offset_1 ? offset_1 : offsetSaved; + cctx->savedRep[1] = offset_2 ? offset_2 : offsetSaved; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_doubleFast(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + const U32 mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_doubleFast_generic(ctx, src, srcSize, 7); return; + } +} + + +static void ZSTD_compressBlock_doubleFast_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 mls) +{ + U32* const hashLong = ctx->hashTable; + U32 const hBitsL = ctx->params.cParams.hashLog; + U32* const hashSmall = ctx->chainTable; + U32 const hBitsS = ctx->params.cParams.chainLog; + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const base = ctx->base; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const dictStart = dictBase + lowestIndex; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const lowPrefixPtr = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + U32 offset_1=ctx->rep[0], offset_2=ctx->rep[1]; + + /* Search Loop */ + while (ip < ilimit) { /* < instead of <=, because (ip+1) */ + const size_t hSmall = ZSTD_hashPtr(ip, hBitsS, mls); + const U32 matchIndex = hashSmall[hSmall]; + const BYTE* matchBase = matchIndex < dictLimit ? dictBase : base; + const BYTE* match = matchBase + matchIndex; + + const size_t hLong = ZSTD_hashPtr(ip, hBitsL, 8); + const U32 matchLongIndex = hashLong[hLong]; + const BYTE* matchLongBase = matchLongIndex < dictLimit ? dictBase : base; + const BYTE* matchLong = matchLongBase + matchLongIndex; + + const U32 current = (U32)(ip-base); + const U32 repIndex = current + 1 - offset_1; /* offset_1 expected <= current +1 */ + const BYTE* repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* repMatch = repBase + repIndex; + size_t mLength; + hashSmall[hSmall] = hashLong[hLong] = current; /* update hash table */ + + if ( (((U32)((dictLimit-1) - repIndex) >= 3) /* intentional underflow */ & (repIndex > lowestIndex)) + && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { + const BYTE* repMatchEnd = repIndex < dictLimit ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, lowPrefixPtr) + 4; + ip++; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, 0, mLength-MINMATCH); + } else { + if ((matchLongIndex > lowestIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { + const BYTE* matchEnd = matchLongIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchLongIndex < dictLimit ? dictStart : lowPrefixPtr; + U32 offset; + mLength = ZSTD_count_2segments(ip+8, matchLong+8, iend, matchEnd, lowPrefixPtr) + 8; + offset = current - matchLongIndex; + while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else if ((matchIndex > lowestIndex) && (MEM_read32(match) == MEM_read32(ip))) { + size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); + U32 const matchIndex3 = hashLong[h3]; + const BYTE* const match3Base = matchIndex3 < dictLimit ? dictBase : base; + const BYTE* match3 = match3Base + matchIndex3; + U32 offset; + hashLong[h3] = current + 1; + if ( (matchIndex3 > lowestIndex) && (MEM_read64(match3) == MEM_read64(ip+1)) ) { + const BYTE* matchEnd = matchIndex3 < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex3 < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+9, match3+8, iend, matchEnd, lowPrefixPtr) + 8; + ip++; + offset = current+1 - matchIndex3; + while (((ip>anchor) & (match3>lowMatchPtr)) && (ip[-1] == match3[-1])) { ip--; match3--; mLength++; } /* catch up */ + } else { + const BYTE* matchEnd = matchIndex < dictLimit ? dictEnd : iend; + const BYTE* lowMatchPtr = matchIndex < dictLimit ? dictStart : lowPrefixPtr; + mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, lowPrefixPtr) + 4; + offset = current - matchIndex; + while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ + } + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStorePtr, ip-anchor, anchor, offset + ZSTD_REP_MOVE, mLength-MINMATCH); + + } else { + ip += ((ip-anchor) >> g_searchStrength) + 1; + continue; + } } + + /* found a match : store it */ + ip += mLength; + anchor = ip; + + if (ip <= ilimit) { + /* Fill Table */ + hashSmall[ZSTD_hashPtr(base+current+2, hBitsS, mls)] = current+2; + hashLong[ZSTD_hashPtr(base+current+2, hBitsL, 8)] = current+2; + hashSmall[ZSTD_hashPtr(ip-2, hBitsS, mls)] = (U32)(ip-2-base); + hashLong[ZSTD_hashPtr(ip-2, hBitsL, 8)] = (U32)(ip-2-base); + /* check immediate repcode */ + while (ip <= ilimit) { + U32 const current2 = (U32)(ip-base); + U32 const repIndex2 = current2 - offset_2; + const BYTE* repMatch2 = repIndex2 < dictLimit ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((dictLimit-1) - repIndex2) >= 3) & (repIndex2 > lowestIndex)) /* intentional overflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + const BYTE* const repEnd2 = repIndex2 < dictLimit ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch2+EQUAL_READ32, iend, repEnd2, lowPrefixPtr) + EQUAL_READ32; + U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, repLength2-MINMATCH); + hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; + hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; + ip += repLength2; + anchor = ip; + continue; + } + break; + } } } + + /* save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_doubleFast_extDict(ZSTD_CCtx* ctx, + const void* src, size_t srcSize) +{ + U32 const mls = ctx->params.cParams.searchLength; + switch(mls) + { + default: + case 4 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 4); return; + case 5 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 5); return; + case 6 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 6); return; + case 7 : + ZSTD_compressBlock_doubleFast_extDict_generic(ctx, src, srcSize, 7); return; + } +} + + +/*-************************************* +* Binary Tree search +***************************************/ +/** ZSTD_insertBt1() : add one or multiple positions to tree. +* ip : assumed <= iend-8 . +* @return : nb of positions added */ +static U32 ZSTD_insertBt1(ZSTD_CCtx* zc, const BYTE* const ip, const U32 mls, const BYTE* const iend, U32 nbCompares, + U32 extDict) +{ + U32* const hashTable = zc->hashTable; + U32 const hashLog = zc->params.cParams.hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + U32 const btLog = zc->params.cParams.chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* match; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = smallerPtr + 1; + U32 dummy32; /* to be nullified at the end */ + U32 const windowLow = zc->lowLimit; + U32 matchEndIdx = current+8; + size_t bestLength = 8; +#ifdef ZSTD_C_PREDICT + U32 predictedSmall = *(bt + 2*((current-1)&btMask) + 0); + U32 predictedLarge = *(bt + 2*((current-1)&btMask) + 1); + predictedSmall += (predictedSmall>0); + predictedLarge += (predictedLarge>0); +#endif /* ZSTD_C_PREDICT */ + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + +#ifdef ZSTD_C_PREDICT /* note : can create issues when hlog small <= 11 */ + const U32* predictPtr = bt + 2*((matchIndex-1) & btMask); /* written this way, as bt is a roll buffer */ + if (matchIndex == predictedSmall) { + /* no need to check length, result known */ + *smallerPtr = matchIndex; + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + predictedSmall = predictPtr[1] + (predictPtr[1]>0); + continue; + } + if (matchIndex == predictedLarge) { + *largerPtr = matchIndex; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + predictedLarge = predictPtr[0] + (predictPtr[0]>0); + continue; + } +#endif + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + bestLength = matchLength; + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + } + + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop , to guarantee consistency ; miss a bit of compression, but other solutions can corrupt the tree */ + + if (match[matchLength] < ip[matchLength]) { /* necessarily within correct buffer */ + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + if (bestLength > 384) return MIN(192, (U32)(bestLength - 384)); /* speed optimization */ + if (matchEndIdx > current + 8) return matchEndIdx - current - 8; + return 1; +} + + +static size_t ZSTD_insertBtAndFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iend, + size_t* offsetPtr, + U32 nbCompares, const U32 mls, + U32 extDict) +{ + U32* const hashTable = zc->hashTable; + U32 const hashLog = zc->params.cParams.hashLog; + size_t const h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const bt = zc->chainTable; + U32 const btLog = zc->params.cParams.chainLog - 1; + U32 const btMask = (1 << btLog) - 1; + U32 matchIndex = hashTable[h]; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 current = (U32)(ip-base); + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + size_t bestLength = 0; + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* const nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iend) +1; + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iend, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) + matchEndIdx = matchIndex + (U32)matchLength; + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(current-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) + bestLength = matchLength, *offsetPtr = ZSTD_REP_MOVE + current - matchIndex; + if (ip+matchLength == iend) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return bestLength; +} + + +static void ZSTD_updateTree(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) + idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 0); +} + +/** ZSTD_BtFindBestMatch() : Tree updater, providing best match */ +static size_t ZSTD_BtFindBestMatch ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 0); +} + + +static size_t ZSTD_BtFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 6 : return ZSTD_BtFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + +static void ZSTD_updateTree_extDict(ZSTD_CCtx* zc, const BYTE* const ip, const BYTE* const iend, const U32 nbCompares, const U32 mls) +{ + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while (idx < target) idx += ZSTD_insertBt1(zc, base+idx, mls, iend, nbCompares, 1); +} + + +/** Tree updater, providing best match */ +static size_t ZSTD_BtFindBestMatch_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndFindBestMatch(zc, ip, iLimit, offsetPtr, maxNbAttempts, mls, 1); +} + + +static size_t ZSTD_BtFindBestMatch_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4); + case 5 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5); + case 6 : return ZSTD_BtFindBestMatch_extDict(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6); + } +} + + + +/* ********************************* +* Hash Chain +***********************************/ +#define NEXT_IN_CHAIN(d, mask) chainTable[(d) & mask] + +/* Update chains up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndex (ZSTD_CCtx* zc, const BYTE* ip, U32 mls) +{ + U32* const hashTable = zc->hashTable; + const U32 hashLog = zc->params.cParams.hashLog; + U32* const chainTable = zc->chainTable; + const U32 chainMask = (1 << zc->params.cParams.chainLog) - 1; + const BYTE* const base = zc->base; + const U32 target = (U32)(ip - base); + U32 idx = zc->nextToUpdate; + + while(idx < target) { /* catch up */ + size_t const h = ZSTD_hashPtr(base+idx, hashLog, mls); + NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; + hashTable[h] = idx; + idx++; + } + + zc->nextToUpdate = target; + return hashTable[ZSTD_hashPtr(ip, hashLog, mls)]; +} + + + +FORCE_INLINE /* inlining is important to hardwire a hot branch (template emulation) */ +size_t ZSTD_HcFindBestMatch_generic ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* const ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 mls, const U32 extDict) +{ + U32* const chainTable = zc->chainTable; + const U32 chainSize = (1 << zc->params.cParams.chainLog); + const U32 chainMask = chainSize-1; + const BYTE* const base = zc->base; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const U32 lowLimit = zc->lowLimit; + const U32 current = (U32)(ip-base); + const U32 minChain = current > chainSize ? current - chainSize : 0; + int nbAttempts=maxNbAttempts; + size_t ml=EQUAL_READ32-1; + + /* HC4 match finder */ + U32 matchIndex = ZSTD_insertAndFindFirstIndex (zc, ip, mls); + + for ( ; (matchIndex>lowLimit) & (nbAttempts>0) ; nbAttempts--) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex >= dictLimit) { + match = base + matchIndex; + if (match[ml] == ip[ml]) /* potentially better */ + currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex; + if (MEM_read32(match) == MEM_read32(ip)) /* assumption : matchIndex <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+EQUAL_READ32, match+EQUAL_READ32, iLimit, dictEnd, prefixStart) + EQUAL_READ32; + } + + /* save best solution */ + if (currentMl > ml) { ml = currentMl; *offsetPtr = current - matchIndex + ZSTD_REP_MOVE; if (ip+currentMl == iLimit) break; /* best possible, and avoid read overflow*/ } + + if (matchIndex <= minChain) break; + matchIndex = NEXT_IN_CHAIN(matchIndex, chainMask); + } + + return ml; +} + + +FORCE_INLINE size_t ZSTD_HcFindBestMatch_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 0); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 0); + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 0); + } +} + + +FORCE_INLINE size_t ZSTD_HcFindBestMatch_extDict_selectMLS ( + ZSTD_CCtx* zc, + const BYTE* ip, const BYTE* const iLimit, + size_t* offsetPtr, + const U32 maxNbAttempts, const U32 matchLengthSearch) +{ + switch(matchLengthSearch) + { + default : + case 4 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 4, 1); + case 5 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 5, 1); + case 6 : return ZSTD_HcFindBestMatch_generic(zc, ip, iLimit, offsetPtr, maxNbAttempts, 6, 1); + } +} + + +/* ******************************* +* Common parser - lazy strategy +*********************************/ +FORCE_INLINE +void ZSTD_compressBlock_lazy_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base + ctx->dictLimit; + + U32 const maxSearches = 1 << ctx->params.cParams.searchLog; + U32 const mls = ctx->params.cParams.searchLength; + + typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, + size_t* offsetPtr, + U32 maxNbAttempts, U32 matchLengthSearch); + searchMax_f const searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS : ZSTD_HcFindBestMatch_selectMLS; + U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1], savedOffset=0; + + /* init */ + ip += (ip==base); + ctx->nextToUpdate3 = ctx->nextToUpdate; + { U32 const maxRep = (U32)(ip-base); + if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; + if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + } + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + + /* check repCode */ + if ((offset_1>0) & (MEM_read32(ip+1) == MEM_read32(ip+1 - offset_1))) { + /* repcode : we take it */ + matchLength = ZSTD_count(ip+1+EQUAL_READ32, ip+1+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + if (depth==0) goto _storeSequence; + } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < EQUAL_READ32) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const mlRep = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + int const gain2 = (int)(mlRep * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((mlRep >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = mlRep, offset = 0, start = ip; + } + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + size_t const ml2 = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_1, iend) + EQUAL_READ32; + int const gain2 = (int)(ml2 * 4); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = ml2, offset = 0, start = ip; + } + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + while ((start>anchor) && (start>base+offset-ZSTD_REP_MOVE) && (start[-1] == start[-1-offset+ZSTD_REP_MOVE])) /* only search for offset within prefix */ + { start--; matchLength++; } + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while ( (ip <= ilimit) + && ((offset_2>0) + & (MEM_read32(ip) == MEM_read32(ip - offset_2)) )) { + /* store sequence */ + matchLength = ZSTD_count(ip+EQUAL_READ32, ip+EQUAL_READ32-offset_2, iend) + EQUAL_READ32; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap repcodes */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } } + + /* Save reps for next block */ + ctx->savedRep[0] = offset_1 ? offset_1 : savedOffset; + ctx->savedRep[1] = offset_2 ? offset_2 : savedOffset; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +static void ZSTD_compressBlock_btlazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 1, 2); +} + +static void ZSTD_compressBlock_lazy2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 2); +} + +static void ZSTD_compressBlock_lazy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 1); +} + +static void ZSTD_compressBlock_greedy(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_generic(ctx, src, srcSize, 0, 0); +} + + +FORCE_INLINE +void ZSTD_compressBlock_lazy_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, + const U32 searchMethod, const U32 depth) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 dictLimit = ctx->dictLimit; + const U32 lowestIndex = ctx->lowLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const dictStart = dictBase + ctx->lowLimit; + + const U32 maxSearches = 1 << ctx->params.cParams.searchLog; + const U32 mls = ctx->params.cParams.searchLength; + + typedef size_t (*searchMax_f)(ZSTD_CCtx* zc, const BYTE* ip, const BYTE* iLimit, + size_t* offsetPtr, + U32 maxNbAttempts, U32 matchLengthSearch); + searchMax_f searchMax = searchMethod ? ZSTD_BtFindBestMatch_selectMLS_extDict : ZSTD_HcFindBestMatch_extDict_selectMLS; + + U32 offset_1 = ctx->rep[0], offset_2 = ctx->rep[1]; + + /* init */ + ctx->nextToUpdate3 = ctx->nextToUpdate; + ip += (ip == prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + size_t matchLength=0; + size_t offset=0; + const BYTE* start=ip+1; + U32 current = (U32)(ip-base); + + /* check repCode */ + { const U32 repIndex = (U32)(current+1 - offset_1); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip+1) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+1+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + if (depth==0) goto _storeSequence; + } } + + /* first search (depth 0) */ + { size_t offsetFound = 99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offsetFound, maxSearches, mls); + if (ml2 > matchLength) + matchLength = ml2, start = ip, offset=offsetFound; + } + + if (matchLength < EQUAL_READ32) { + ip += ((ip-anchor) >> g_searchStrength) + 1; /* jump faster over incompressible sections */ + continue; + } + + /* let's try to find a better solution */ + if (depth>=1) + while (ip= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t const repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + int const gain2 = (int)(repLength * 3); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 1 */ + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 4); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; /* search a better one */ + } } + + /* let's find an even better one */ + if ((depth==2) && (ip= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + size_t repLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + int gain2 = (int)(repLength * 4); + int gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 1); + if ((repLength >= EQUAL_READ32) && (gain2 > gain1)) + matchLength = repLength, offset = 0, start = ip; + } } + + /* search match, depth 2 */ + { size_t offset2=99999999; + size_t const ml2 = searchMax(ctx, ip, iend, &offset2, maxSearches, mls); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)offset2+1)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offset+1) + 7); + if ((ml2 >= EQUAL_READ32) && (gain2 > gain1)) { + matchLength = ml2, offset = offset2, start = ip; + continue; + } } } + break; /* nothing found : store previous solution */ + } + + /* catch up */ + if (offset) { + U32 const matchIndex = (U32)((start-base) - (offset - ZSTD_REP_MOVE)); + const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; + const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; + while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ + offset_2 = offset_1; offset_1 = (U32)(offset - ZSTD_REP_MOVE); + } + + /* store sequence */ +_storeSequence: + { size_t const litLength = start - anchor; + ZSTD_storeSeq(seqStorePtr, litLength, anchor, (U32)offset, matchLength-MINMATCH); + anchor = ip = start + matchLength; + } + + /* check immediate repcode */ + while (ip <= ilimit) { + const U32 repIndex = (U32)((ip-base) - offset_2); + const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; + const BYTE* const repMatch = repBase + repIndex; + if (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex > lowestIndex)) /* intentional overflow */ + if (MEM_read32(ip) == MEM_read32(repMatch)) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + matchLength = ZSTD_count_2segments(ip+EQUAL_READ32, repMatch+EQUAL_READ32, iend, repEnd, prefixStart) + EQUAL_READ32; + offset = offset_2; offset_2 = offset_1; offset_1 = (U32)offset; /* swap offset history */ + ZSTD_storeSeq(seqStorePtr, 0, anchor, 0, matchLength-MINMATCH); + ip += matchLength; + anchor = ip; + continue; /* faster when present ... (?) */ + } + break; + } } + + /* Save reps for next block */ + ctx->savedRep[0] = offset_1; ctx->savedRep[1] = offset_2; + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +void ZSTD_compressBlock_greedy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 0); +} + +static void ZSTD_compressBlock_lazy_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 1); +} + +static void ZSTD_compressBlock_lazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 0, 2); +} + +static void ZSTD_compressBlock_btlazy2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ + ZSTD_compressBlock_lazy_extDict_generic(ctx, src, srcSize, 1, 2); +} + + +/* The optimal parser */ +#include "zstd_opt.h" + +static void ZSTD_compressBlock_btopt(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 0); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + +static void ZSTD_compressBlock_btopt2(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_generic(ctx, src, srcSize, 1); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + +static void ZSTD_compressBlock_btopt_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 0); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + +static void ZSTD_compressBlock_btopt2_extDict(ZSTD_CCtx* ctx, const void* src, size_t srcSize) +{ +#ifdef ZSTD_OPT_H_91842398743 + ZSTD_compressBlock_opt_extDict_generic(ctx, src, srcSize, 1); +#else + (void)ctx; (void)src; (void)srcSize; + return; +#endif +} + + +typedef void (*ZSTD_blockCompressor) (ZSTD_CCtx* ctx, const void* src, size_t srcSize); + +static ZSTD_blockCompressor ZSTD_selectBlockCompressor(ZSTD_strategy strat, int extDict) +{ + static const ZSTD_blockCompressor blockCompressor[2][8] = { + { ZSTD_compressBlock_fast, ZSTD_compressBlock_doubleFast, ZSTD_compressBlock_greedy, ZSTD_compressBlock_lazy, ZSTD_compressBlock_lazy2, ZSTD_compressBlock_btlazy2, ZSTD_compressBlock_btopt, ZSTD_compressBlock_btopt2 }, + { ZSTD_compressBlock_fast_extDict, ZSTD_compressBlock_doubleFast_extDict, ZSTD_compressBlock_greedy_extDict, ZSTD_compressBlock_lazy_extDict,ZSTD_compressBlock_lazy2_extDict, ZSTD_compressBlock_btlazy2_extDict, ZSTD_compressBlock_btopt_extDict, ZSTD_compressBlock_btopt2_extDict } + }; + + return blockCompressor[extDict][(U32)strat]; +} + + +static size_t ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->params.cParams.strategy, zc->lowLimit < zc->dictLimit); + const BYTE* const base = zc->base; + const BYTE* const istart = (const BYTE*)src; + const U32 current = (U32)(istart-base); + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) return 0; /* don't even attempt compression below a certain srcSize */ + ZSTD_resetSeqStore(&(zc->seqStore)); + if (current > zc->nextToUpdate + 384) + zc->nextToUpdate = current - MIN(192, (U32)(current - zc->nextToUpdate - 384)); /* update tree not updated after finding very long rep matches */ + blockCompressor(zc, src, srcSize); + return ZSTD_compressSequences(zc, dst, dstCapacity, srcSize); +} + + +/*! ZSTD_compress_generic() : +* Compress a chunk of data into one or multiple blocks. +* All blocks will be terminated, all input will be consumed. +* Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. +* Frame is supposed already started (header already produced) +* @return : compressed size, or an error code +*/ +static size_t ZSTD_compress_generic (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 lastFrameChunk) +{ + size_t blockSize = cctx->blockSize; + size_t remaining = srcSize; + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + U32 const maxDist = 1 << cctx->params.cParams.windowLog; + + if (cctx->params.fParams.checksumFlag && srcSize) + XXH64_update(&cctx->xxhState, src, srcSize); + + while (remaining) { + U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); + size_t cSize; + + if (dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE) return ERROR(dstSize_tooSmall); /* not enough space to store compressed block */ + if (remaining < blockSize) blockSize = remaining; + + /* preemptive overflow correction */ + if (cctx->lowLimit > (2U<<30)) { + U32 const cycleMask = (1 << ZSTD_cycleLog(cctx->params.cParams.hashLog, cctx->params.cParams.strategy)) - 1; + U32 const current = (U32)(ip - cctx->base); + U32 const newCurrent = (current & cycleMask) + (1 << cctx->params.cParams.windowLog); + U32 const correction = current - newCurrent; + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX_64 <= 30); + ZSTD_reduceIndex(cctx, correction); + cctx->base += correction; + cctx->dictBase += correction; + cctx->lowLimit -= correction; + cctx->dictLimit -= correction; + if (cctx->nextToUpdate < correction) cctx->nextToUpdate = 0; + else cctx->nextToUpdate -= correction; + } + + if ((U32)(ip+blockSize - cctx->base) > cctx->loadedDictEnd + maxDist) { + /* enforce maxDist */ + U32 const newLowLimit = (U32)(ip+blockSize - cctx->base) - maxDist; + if (cctx->lowLimit < newLowLimit) cctx->lowLimit = newLowLimit; + if (cctx->dictLimit < cctx->lowLimit) cctx->dictLimit = cctx->lowLimit; + } + + cSize = ZSTD_compressBlock_internal(cctx, op+ZSTD_blockHeaderSize, dstCapacity-ZSTD_blockHeaderSize, ip, blockSize); + if (ZSTD_isError(cSize)) return cSize; + + if (cSize == 0) { /* block is not compressible */ + U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(blockSize << 3); + if (blockSize + ZSTD_blockHeaderSize > dstCapacity) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); /* no pb, 4th byte will be overwritten */ + memcpy(op + ZSTD_blockHeaderSize, ip, blockSize); + cSize = ZSTD_blockHeaderSize+blockSize; + } else { + U32 const cBlockHeader24 = lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); + MEM_writeLE24(op, cBlockHeader24); + cSize += ZSTD_blockHeaderSize; + } + + remaining -= blockSize; + dstCapacity -= cSize; + ip += blockSize; + op += cSize; + } + + if (lastFrameChunk && (op>ostart)) cctx->stage = ZSTDcs_ending; + return op-ostart; +} + + +static size_t ZSTD_writeFrameHeader(void* dst, size_t dstCapacity, + ZSTD_parameters params, U64 pledgedSrcSize, U32 dictID) +{ BYTE* const op = (BYTE*)dst; + U32 const dictIDSizeCode = (dictID>0) + (dictID>=256) + (dictID>=65536); /* 0-3 */ + U32 const checksumFlag = params.fParams.checksumFlag>0; + U32 const windowSize = 1U << params.cParams.windowLog; + U32 const singleSegment = params.fParams.contentSizeFlag && (windowSize > (pledgedSrcSize-1)); + BYTE const windowLogByte = (BYTE)((params.cParams.windowLog - ZSTD_WINDOWLOG_ABSOLUTEMIN) << 3); + U32 const fcsCode = params.fParams.contentSizeFlag ? + (pledgedSrcSize>=256) + (pledgedSrcSize>=65536+256) + (pledgedSrcSize>=0xFFFFFFFFU) : /* 0-3 */ + 0; + BYTE const frameHeaderDecriptionByte = (BYTE)(dictIDSizeCode + (checksumFlag<<2) + (singleSegment<<5) + (fcsCode<<6) ); + size_t pos; + + if (dstCapacity < ZSTD_frameHeaderSize_max) return ERROR(dstSize_tooSmall); + + MEM_writeLE32(dst, ZSTD_MAGICNUMBER); + op[4] = frameHeaderDecriptionByte; pos=5; + if (!singleSegment) op[pos++] = windowLogByte; + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : op[pos] = (BYTE)(dictID); pos++; break; + case 2 : MEM_writeLE16(op+pos, (U16)dictID); pos+=2; break; + case 3 : MEM_writeLE32(op+pos, dictID); pos+=4; break; + } + switch(fcsCode) + { + default: /* impossible */ + case 0 : if (singleSegment) op[pos++] = (BYTE)(pledgedSrcSize); break; + case 1 : MEM_writeLE16(op+pos, (U16)(pledgedSrcSize-256)); pos+=2; break; + case 2 : MEM_writeLE32(op+pos, (U32)(pledgedSrcSize)); pos+=4; break; + case 3 : MEM_writeLE64(op+pos, (U64)(pledgedSrcSize)); pos+=8; break; + } + return pos; +} + + +static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + U32 frame, U32 lastFrameChunk) +{ + const BYTE* const ip = (const BYTE*) src; + size_t fhSize = 0; + + if (cctx->stage==ZSTDcs_created) return ERROR(stage_wrong); /* missing init (ZSTD_compressBegin) */ + + if (frame && (cctx->stage==ZSTDcs_init)) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, cctx->frameContentSize, cctx->dictID); + if (ZSTD_isError(fhSize)) return fhSize; + dstCapacity -= fhSize; + dst = (char*)dst + fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + /* Check if blocks follow each other */ + if (src != cctx->nextSrc) { + /* not contiguous */ + ptrdiff_t const delta = cctx->nextSrc - ip; + cctx->lowLimit = cctx->dictLimit; + cctx->dictLimit = (U32)(cctx->nextSrc - cctx->base); + cctx->dictBase = cctx->base; + cctx->base -= delta; + cctx->nextToUpdate = cctx->dictLimit; + if (cctx->dictLimit - cctx->lowLimit < HASH_READ_SIZE) cctx->lowLimit = cctx->dictLimit; /* too small extDict */ + } + + /* if input and dictionary overlap : reduce dictionary (area presumed modified by input) */ + if ((ip+srcSize > cctx->dictBase + cctx->lowLimit) & (ip < cctx->dictBase + cctx->dictLimit)) { + ptrdiff_t const highInputIdx = (ip + srcSize) - cctx->dictBase; + U32 const lowLimitMax = (highInputIdx > (ptrdiff_t)cctx->dictLimit) ? cctx->dictLimit : (U32)highInputIdx; + cctx->lowLimit = lowLimitMax; + } + + cctx->nextSrc = ip + srcSize; + + { size_t const cSize = frame ? + ZSTD_compress_generic (cctx, dst, dstCapacity, src, srcSize, lastFrameChunk) : + ZSTD_compressBlock_internal (cctx, dst, dstCapacity, src, srcSize); + if (ZSTD_isError(cSize)) return cSize; + return cSize + fhSize; + } +} + + +size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 0); +} + + +size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx) +{ + return MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << cctx->params.cParams.windowLog); +} + +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const blockSizeMax = ZSTD_getBlockSizeMax(cctx); + if (srcSize > blockSizeMax) return ERROR(srcSize_wrong); + return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0, 0); +} + + +static size_t ZSTD_loadDictionaryContent(ZSTD_CCtx* zc, const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*) src; + const BYTE* const iend = ip + srcSize; + + /* input becomes current prefix */ + zc->lowLimit = zc->dictLimit; + zc->dictLimit = (U32)(zc->nextSrc - zc->base); + zc->dictBase = zc->base; + zc->base += ip - zc->nextSrc; + zc->nextToUpdate = zc->dictLimit; + zc->loadedDictEnd = (U32)(iend - zc->base); + + zc->nextSrc = iend; + if (srcSize <= HASH_READ_SIZE) return 0; + + switch(zc->params.cParams.strategy) + { + case ZSTD_fast: + ZSTD_fillHashTable (zc, iend, zc->params.cParams.searchLength); + break; + + case ZSTD_dfast: + ZSTD_fillDoubleHashTable (zc, iend, zc->params.cParams.searchLength); + break; + + case ZSTD_greedy: + case ZSTD_lazy: + case ZSTD_lazy2: + ZSTD_insertAndFindFirstIndex (zc, iend-HASH_READ_SIZE, zc->params.cParams.searchLength); + break; + + case ZSTD_btlazy2: + case ZSTD_btopt: + case ZSTD_btopt2: + ZSTD_updateTree(zc, iend-HASH_READ_SIZE, iend, 1 << zc->params.cParams.searchLog, zc->params.cParams.searchLength); + break; + + default: + return ERROR(GENERIC); /* strategy doesn't exist; impossible */ + } + + zc->nextToUpdate = zc->loadedDictEnd; + return 0; +} + + +/* Dictionaries that assign zero probability to symbols that show up causes problems + when FSE encoding. Refuse dictionaries that assign zero probability to symbols + that we may encounter during compression. + NOTE: This behavior is not standard and could be improved in the future. */ +static size_t ZSTD_checkDictNCount(short* normalizedCounter, unsigned dictMaxSymbolValue, unsigned maxSymbolValue) { + U32 s; + if (dictMaxSymbolValue < maxSymbolValue) return ERROR(dictionary_corrupted); + for (s = 0; s <= maxSymbolValue; ++s) { + if (normalizedCounter[s] == 0) return ERROR(dictionary_corrupted); + } + return 0; +} + + +/* Dictionary format : + Magic == ZSTD_DICT_MAGIC (4 bytes) + HUF_writeCTable(256) + FSE_writeNCount(off) + FSE_writeNCount(ml) + FSE_writeNCount(ll) + RepOffsets + Dictionary content +*/ +/*! ZSTD_loadDictEntropyStats() : + @return : size read from dictionary + note : magic number supposed already checked */ +static size_t ZSTD_loadDictEntropyStats(ZSTD_CCtx* cctx, const void* dict, size_t dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + short offcodeNCount[MaxOff+1]; + unsigned offcodeMaxValue = MaxOff; + BYTE scratchBuffer[1<hufTable, 255, dict, dictSize); + if (HUF_isError(hufHeaderSize)) return ERROR(dictionary_corrupted); + dictPtr += hufHeaderSize; + } + + { unsigned offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + /* Defer checking offcodeMaxValue because we need to know the size of the dictionary content */ + CHECK_E (FSE_buildCTable_wksp(cctx->offcodeCTable, offcodeNCount, offcodeMaxValue, offcodeLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + /* Every match length code must have non-zero probability */ + CHECK_F (ZSTD_checkDictNCount(matchlengthNCount, matchlengthMaxValue, MaxML)); + CHECK_E (FSE_buildCTable_wksp(cctx->matchlengthCTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + /* Every literal length code must have non-zero probability */ + CHECK_F (ZSTD_checkDictNCount(litlengthNCount, litlengthMaxValue, MaxLL)); + CHECK_E(FSE_buildCTable_wksp(cctx->litlengthCTable, litlengthNCount, litlengthMaxValue, litlengthLog, scratchBuffer, sizeof(scratchBuffer)), dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + cctx->rep[0] = MEM_readLE32(dictPtr+0); if (cctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[1] = MEM_readLE32(dictPtr+4); if (cctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + cctx->rep[2] = MEM_readLE32(dictPtr+8); if (cctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dictPtr += 12; + + { U32 offcodeMax = MaxOff; + if ((size_t)(dictEnd - dictPtr) <= ((U32)-1) - 128 KB) { + U32 const maxOffset = (U32)(dictEnd - dictPtr) + 128 KB; /* The maximum offset that must be supported */ + /* Calculate minimum offset code required to represent maxOffset */ + offcodeMax = ZSTD_highbit32(maxOffset); + } + /* Every possible supported offset <= dictContentSize + 128 KB must be representable */ + CHECK_F (ZSTD_checkDictNCount(offcodeNCount, offcodeMaxValue, MIN(offcodeMax, MaxOff))); + } + + cctx->flagStaticTables = 1; + return dictPtr - (const BYTE*)dict; +} + +/** ZSTD_compress_insertDictionary() : +* @return : 0, or an error code */ +static size_t ZSTD_compress_insertDictionary(ZSTD_CCtx* zc, const void* dict, size_t dictSize) +{ + if ((dict==NULL) || (dictSize<=8)) return 0; + + /* default : dict is pure content */ + if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return ZSTD_loadDictionaryContent(zc, dict, dictSize); + zc->dictID = zc->params.fParams.noDictIDFlag ? 0 : MEM_readLE32((const char*)dict+4); + + /* known magic number : dict is parsed for entropy stats and content */ + { size_t const loadError = ZSTD_loadDictEntropyStats(zc, (const char*)dict+8 /* skip dictHeader */, dictSize-8); + size_t const eSize = loadError + 8; + if (ZSTD_isError(loadError)) return loadError; + return ZSTD_loadDictionaryContent(zc, (const char*)dict+eSize, dictSize-eSize); + } +} + + +/*! ZSTD_compressBegin_internal() : +* @return : 0, or an error code */ +static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, U64 pledgedSrcSize) +{ + ZSTD_compResetPolicy_e const crp = dictSize ? ZSTDcrp_fullReset : ZSTDcrp_continue; + CHECK_F(ZSTD_resetCCtx_advanced(cctx, params, pledgedSrcSize, crp)); + return ZSTD_compress_insertDictionary(cctx, dict, dictSize); +} + + +/*! ZSTD_compressBegin_advanced() : +* @return : 0, or an error code */ +size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* compression parameters verification and optimization */ + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, pledgedSrcSize); +} + + +size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZSTD_compressBegin_internal(cctx, dict, dictSize, params, 0); +} + + +size_t ZSTD_compressBegin(ZSTD_CCtx* zc, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict(zc, NULL, 0, compressionLevel); +} + + +/*! ZSTD_writeEpilogue() : +* Ends a frame. +* @return : nb of bytes written into dst (or an error code) */ +static size_t ZSTD_writeEpilogue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity) +{ + BYTE* const ostart = (BYTE*)dst; + BYTE* op = ostart; + size_t fhSize = 0; + + if (cctx->stage == ZSTDcs_created) return ERROR(stage_wrong); /* init missing */ + + /* special case : empty frame */ + if (cctx->stage == ZSTDcs_init) { + fhSize = ZSTD_writeFrameHeader(dst, dstCapacity, cctx->params, 0, 0); + if (ZSTD_isError(fhSize)) return fhSize; + dstCapacity -= fhSize; + op += fhSize; + cctx->stage = ZSTDcs_ongoing; + } + + if (cctx->stage != ZSTDcs_ending) { + /* write one last empty block, make it the "last" block */ + U32 const cBlockHeader24 = 1 /* last block */ + (((U32)bt_raw)<<1) + 0; + if (dstCapacity<4) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, cBlockHeader24); + op += ZSTD_blockHeaderSize; + dstCapacity -= ZSTD_blockHeaderSize; + } + + if (cctx->params.fParams.checksumFlag) { + U32 const checksum = (U32) XXH64_digest(&cctx->xxhState); + if (dstCapacity<4) return ERROR(dstSize_tooSmall); + MEM_writeLE32(op, checksum); + op += 4; + } + + cctx->stage = ZSTDcs_created; /* return to "created but no init" status */ + return op-ostart; +} + + +size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t endResult; + size_t const cSize = ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1, 1); + if (ZSTD_isError(cSize)) return cSize; + endResult = ZSTD_writeEpilogue(cctx, (char*)dst + cSize, dstCapacity-cSize); + if (ZSTD_isError(endResult)) return endResult; + return cSize + endResult; +} + + +static size_t ZSTD_compress_internal (ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + CHECK_F(ZSTD_compressBegin_internal(cctx, dict, dictSize, params, srcSize)); + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + +size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params) +{ + CHECK_F(ZSTD_checkCParams(params.cParams)); + return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters params = ZSTD_getParams(compressionLevel, srcSize, dict ? dictSize : 0); + params.fParams.contentSizeFlag = 1; + return ZSTD_compress_internal(ctx, dst, dstCapacity, src, srcSize, dict, dictSize, params); +} + +size_t ZSTD_compressCCtx (ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + return ZSTD_compress_usingDict(ctx, dst, dstCapacity, src, srcSize, NULL, 0, compressionLevel); +} + +size_t ZSTD_compress(void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel) +{ + size_t result; + ZSTD_CCtx ctxBody; + memset(&ctxBody, 0, sizeof(ctxBody)); + memcpy(&ctxBody.customMem, &defaultCustomMem, sizeof(ZSTD_customMem)); + result = ZSTD_compressCCtx(&ctxBody, dst, dstCapacity, src, srcSize, compressionLevel); + ZSTD_free(ctxBody.workSpace, defaultCustomMem); /* can't free ctxBody itself, as it's on stack; free only heap content */ + return result; +} + + +/* ===== Dictionary API ===== */ + +struct ZSTD_CDict_s { + void* dictContent; + size_t dictContentSize; + ZSTD_CCtx* refContext; +}; /* typedef'd tp ZSTD_CDict within "zstd.h" */ + +size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support sizeof on NULL */ + return ZSTD_sizeof_CCtx(cdict->refContext) + cdict->dictContentSize; +} + +ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, ZSTD_parameters params, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + { ZSTD_CDict* const cdict = (ZSTD_CDict*) ZSTD_malloc(sizeof(ZSTD_CDict), customMem); + void* const dictContent = ZSTD_malloc(dictSize, customMem); + ZSTD_CCtx* const cctx = ZSTD_createCCtx_advanced(customMem); + + if (!dictContent || !cdict || !cctx) { + ZSTD_free(dictContent, customMem); + ZSTD_free(cdict, customMem); + ZSTD_free(cctx, customMem); + return NULL; + } + + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } + { size_t const errorCode = ZSTD_compressBegin_advanced(cctx, dictContent, dictSize, params, 0); + if (ZSTD_isError(errorCode)) { + ZSTD_free(dictContent, customMem); + ZSTD_free(cdict, customMem); + ZSTD_free(cctx, customMem); + return NULL; + } } + + cdict->dictContent = dictContent; + cdict->dictContentSize = dictSize; + cdict->refContext = cctx; + return cdict; + } +} + +ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, dictSize); + params.fParams.contentSizeFlag = 1; + return ZSTD_createCDict_advanced(dict, dictSize, params, allocator); +} + +size_t ZSTD_freeCDict(ZSTD_CDict* cdict) +{ + if (cdict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = cdict->refContext->customMem; + ZSTD_freeCCtx(cdict->refContext); + ZSTD_free(cdict->dictContent, cMem); + ZSTD_free(cdict, cMem); + return 0; + } +} + +static ZSTD_parameters ZSTD_getParamsFromCDict(const ZSTD_CDict* cdict) { + return ZSTD_getParamsFromCCtx(cdict->refContext); +} + +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, U64 pledgedSrcSize) +{ + if (cdict->dictContentSize) CHECK_F(ZSTD_copyCCtx(cctx, cdict->refContext, pledgedSrcSize)) + else CHECK_F(ZSTD_compressBegin_advanced(cctx, NULL, 0, cdict->refContext->params, pledgedSrcSize)); + return 0; +} + +/*! ZSTD_compress_usingCDict() : +* Compression using a digested Dictionary. +* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. +* Note that compression level is decided during dictionary creation */ +size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict) +{ + CHECK_F(ZSTD_compressBegin_usingCDict(cctx, cdict, srcSize)); + + if (cdict->refContext->params.fParams.contentSizeFlag==1) { + cctx->params.fParams.contentSizeFlag = 1; + cctx->frameContentSize = srcSize; + } + + return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); +} + + + +/* ****************************************************************** +* Streaming +********************************************************************/ + +typedef enum { zcss_init, zcss_load, zcss_flush, zcss_final } ZSTD_cStreamStage; + +struct ZSTD_CStream_s { + ZSTD_CCtx* cctx; + ZSTD_CDict* cdictLocal; + const ZSTD_CDict* cdict; + char* inBuff; + size_t inBuffSize; + size_t inToCompress; + size_t inBuffPos; + size_t inBuffTarget; + size_t blockSize; + char* outBuff; + size_t outBuffSize; + size_t outBuffContentSize; + size_t outBuffFlushedSize; + ZSTD_cStreamStage stage; + U32 checksum; + U32 frameEnded; + U64 pledgedSrcSize; + U64 inputProcessed; + ZSTD_parameters params; + ZSTD_customMem customMem; +}; /* typedef'd to ZSTD_CStream within "zstd.h" */ + +ZSTD_CStream* ZSTD_createCStream(void) +{ + return ZSTD_createCStream_advanced(defaultCustomMem); +} + +ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_CStream* zcs; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + zcs = (ZSTD_CStream*)ZSTD_malloc(sizeof(ZSTD_CStream), customMem); + if (zcs==NULL) return NULL; + memset(zcs, 0, sizeof(ZSTD_CStream)); + memcpy(&zcs->customMem, &customMem, sizeof(ZSTD_customMem)); + zcs->cctx = ZSTD_createCCtx_advanced(customMem); + if (zcs->cctx == NULL) { ZSTD_freeCStream(zcs); return NULL; } + return zcs; +} + +size_t ZSTD_freeCStream(ZSTD_CStream* zcs) +{ + if (zcs==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = zcs->customMem; + ZSTD_freeCCtx(zcs->cctx); + ZSTD_freeCDict(zcs->cdictLocal); + ZSTD_free(zcs->inBuff, cMem); + ZSTD_free(zcs->outBuff, cMem); + ZSTD_free(zcs, cMem); + return 0; + } +} + + +/*====== Initialization ======*/ + +size_t ZSTD_CStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } +size_t ZSTD_CStreamOutSize(void) { return ZSTD_compressBound(ZSTD_BLOCKSIZE_ABSOLUTEMAX) + ZSTD_blockHeaderSize + 4 /* 32-bits hash */ ; } + +size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize) +{ + if (zcs->inBuffSize==0) return ERROR(stage_wrong); /* zcs has not been init at least once */ + + if (zcs->cdict) CHECK_F(ZSTD_compressBegin_usingCDict(zcs->cctx, zcs->cdict, pledgedSrcSize)) + else CHECK_F(ZSTD_compressBegin_advanced(zcs->cctx, NULL, 0, zcs->params, pledgedSrcSize)); + + zcs->inToCompress = 0; + zcs->inBuffPos = 0; + zcs->inBuffTarget = zcs->blockSize; + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + zcs->frameEnded = 0; + zcs->pledgedSrcSize = pledgedSrcSize; + zcs->inputProcessed = 0; + return 0; /* ready to go */ +} + +size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, + const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize) +{ + /* allocate buffers */ + { size_t const neededInBuffSize = (size_t)1 << params.cParams.windowLog; + if (zcs->inBuffSize < neededInBuffSize) { + zcs->inBuffSize = neededInBuffSize; + ZSTD_free(zcs->inBuff, zcs->customMem); + zcs->inBuff = (char*) ZSTD_malloc(neededInBuffSize, zcs->customMem); + if (zcs->inBuff == NULL) return ERROR(memory_allocation); + } + zcs->blockSize = MIN(ZSTD_BLOCKSIZE_ABSOLUTEMAX, neededInBuffSize); + } + if (zcs->outBuffSize < ZSTD_compressBound(zcs->blockSize)+1) { + zcs->outBuffSize = ZSTD_compressBound(zcs->blockSize)+1; + ZSTD_free(zcs->outBuff, zcs->customMem); + zcs->outBuff = (char*) ZSTD_malloc(zcs->outBuffSize, zcs->customMem); + if (zcs->outBuff == NULL) return ERROR(memory_allocation); + } + + if (dict) { + ZSTD_freeCDict(zcs->cdictLocal); + zcs->cdictLocal = ZSTD_createCDict_advanced(dict, dictSize, params, zcs->customMem); + if (zcs->cdictLocal == NULL) return ERROR(memory_allocation); + zcs->cdict = zcs->cdictLocal; + } else zcs->cdict = NULL; + + zcs->checksum = params.fParams.checksumFlag > 0; + zcs->params = params; + + return ZSTD_resetCStream(zcs, pledgedSrcSize); +} + +/* note : cdict must outlive compression session */ +size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict) +{ + ZSTD_parameters const params = ZSTD_getParamsFromCDict(cdict); + size_t const initError = ZSTD_initCStream_advanced(zcs, NULL, 0, params, 0); + zcs->cdict = cdict; + return initError; +} + +size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, 0, dictSize); + return ZSTD_initCStream_advanced(zcs, dict, dictSize, params, 0); +} + +size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize) +{ + ZSTD_parameters const params = ZSTD_getParams(compressionLevel, pledgedSrcSize, 0); + return ZSTD_initCStream_advanced(zcs, NULL, 0, params, pledgedSrcSize); +} + +size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) +{ + return ZSTD_initCStream_usingDict(zcs, NULL, 0, compressionLevel); +} + +size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs) +{ + if (zcs==NULL) return 0; /* support sizeof on NULL */ + return sizeof(zcs) + ZSTD_sizeof_CCtx(zcs->cctx) + ZSTD_sizeof_CDict(zcs->cdictLocal) + zcs->outBuffSize + zcs->inBuffSize; +} + +/*====== Compression ======*/ + +typedef enum { zsf_gather, zsf_flush, zsf_end } ZSTD_flush_e; + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + +static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, + void* dst, size_t* dstCapacityPtr, + const void* src, size_t* srcSizePtr, + ZSTD_flush_e const flush) +{ + U32 someMoreWork = 1; + const char* const istart = (const char*)src; + const char* const iend = istart + *srcSizePtr; + const char* ip = istart; + char* const ostart = (char*)dst; + char* const oend = ostart + *dstCapacityPtr; + char* op = ostart; + + while (someMoreWork) { + switch(zcs->stage) + { + case zcss_init: return ERROR(init_missing); /* call ZBUFF_compressInit() first ! */ + + case zcss_load: + /* complete inBuffer */ + { size_t const toLoad = zcs->inBuffTarget - zcs->inBuffPos; + size_t const loaded = ZSTD_limitCopy(zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); + zcs->inBuffPos += loaded; + ip += loaded; + if ( (zcs->inBuffPos==zcs->inToCompress) || (!flush && (toLoad != loaded)) ) { + someMoreWork = 0; break; /* not enough input to get a full block : stop there, wait for more */ + } } + /* compress current block (note : this stage cannot be stopped in the middle) */ + { void* cDst; + size_t cSize; + size_t const iSize = zcs->inBuffPos - zcs->inToCompress; + size_t oSize = oend-op; + if (oSize >= ZSTD_compressBound(iSize)) + cDst = op; /* compress directly into output buffer (avoid flush stage) */ + else + cDst = zcs->outBuff, oSize = zcs->outBuffSize; + cSize = (flush == zsf_end) ? + ZSTD_compressEnd(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : + ZSTD_compressContinue(zcs->cctx, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); + if (ZSTD_isError(cSize)) return cSize; + if (flush == zsf_end) zcs->frameEnded = 1; + /* prepare next block */ + zcs->inBuffTarget = zcs->inBuffPos + zcs->blockSize; + if (zcs->inBuffTarget > zcs->inBuffSize) + zcs->inBuffPos = 0, zcs->inBuffTarget = zcs->blockSize; /* note : inBuffSize >= blockSize */ + zcs->inToCompress = zcs->inBuffPos; + if (cDst == op) { op += cSize; break; } /* no need to flush */ + zcs->outBuffContentSize = cSize; + zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_flush; /* pass-through to flush stage */ + } + + case zcss_flush: + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + if (toFlush!=flushed) { someMoreWork = 0; break; } /* dst too small to store flushed data : stop there */ + zcs->outBuffContentSize = zcs->outBuffFlushedSize = 0; + zcs->stage = zcss_load; + break; + } + + case zcss_final: + someMoreWork = 0; /* do nothing */ + break; + + default: + return ERROR(GENERIC); /* impossible */ + } + } + + *srcSizePtr = ip - istart; + *dstCapacityPtr = op - ostart; + zcs->inputProcessed += *srcSizePtr; + if (zcs->frameEnded) return 0; + { size_t hintInSize = zcs->inBuffTarget - zcs->inBuffPos; + if (hintInSize==0) hintInSize = zcs->blockSize; + return hintInSize; + } +} + +size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + size_t sizeRead = input->size - input->pos; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + (const char*)(input->src) + input->pos, &sizeRead, zsf_gather); + input->pos += sizeRead; + output->pos += sizeWritten; + return result; +} + + +/*====== Finalize ======*/ + +/*! ZSTD_flushStream() : +* @return : amount of data remaining to flush */ +size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + size_t srcSize = 0; + size_t sizeWritten = output->size - output->pos; + size_t const result = ZSTD_compressStream_generic(zcs, + (char*)(output->dst) + output->pos, &sizeWritten, + &srcSize, &srcSize, /* use a valid src address instead of NULL */ + zsf_flush); + output->pos += sizeWritten; + if (ZSTD_isError(result)) return result; + return zcs->outBuffContentSize - zcs->outBuffFlushedSize; /* remaining to flush */ +} + + +size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) +{ + BYTE* const ostart = (BYTE*)(output->dst) + output->pos; + BYTE* const oend = (BYTE*)(output->dst) + output->size; + BYTE* op = ostart; + + if ((zcs->pledgedSrcSize) && (zcs->inputProcessed != zcs->pledgedSrcSize)) + return ERROR(srcSize_wrong); /* pledgedSrcSize not respected */ + + if (zcs->stage != zcss_final) { + /* flush whatever remains */ + size_t srcSize = 0; + size_t sizeWritten = output->size - output->pos; + size_t const notEnded = ZSTD_compressStream_generic(zcs, ostart, &sizeWritten, &srcSize, &srcSize, zsf_end); /* use a valid src address instead of NULL */ + size_t const remainingToFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + op += sizeWritten; + if (remainingToFlush) { + output->pos += sizeWritten; + return remainingToFlush + ZSTD_BLOCKHEADERSIZE /* final empty block */ + (zcs->checksum * 4); + } + /* create epilogue */ + zcs->stage = zcss_final; + zcs->outBuffContentSize = !notEnded ? 0 : + ZSTD_compressEnd(zcs->cctx, zcs->outBuff, zcs->outBuffSize, NULL, 0); /* write epilogue, including final empty block, into outBuff */ + } + + /* flush epilogue */ + { size_t const toFlush = zcs->outBuffContentSize - zcs->outBuffFlushedSize; + size_t const flushed = ZSTD_limitCopy(op, oend-op, zcs->outBuff + zcs->outBuffFlushedSize, toFlush); + op += flushed; + zcs->outBuffFlushedSize += flushed; + output->pos += op-ostart; + if (toFlush==flushed) zcs->stage = zcss_init; /* end reached */ + return toFlush - flushed; + } +} + + + +/*-===== Pre-defined compression levels =====-*/ + +#define ZSTD_DEFAULT_CLEVEL 1 +#define ZSTD_MAX_CLEVEL 22 +int ZSTD_maxCLevel(void) { return ZSTD_MAX_CLEVEL; } + +static const ZSTD_compressionParameters ZSTD_defaultCParameters[4][ZSTD_MAX_CLEVEL+1] = { +{ /* "default" */ + /* W, C, H, S, L, TL, strat */ + { 18, 12, 12, 1, 7, 16, ZSTD_fast }, /* level 0 - never used */ + { 19, 13, 14, 1, 7, 16, ZSTD_fast }, /* level 1 */ + { 19, 15, 16, 1, 6, 16, ZSTD_fast }, /* level 2 */ + { 20, 16, 17, 1, 5, 16, ZSTD_dfast }, /* level 3.*/ + { 20, 18, 18, 1, 5, 16, ZSTD_dfast }, /* level 4.*/ + { 20, 15, 18, 3, 5, 16, ZSTD_greedy }, /* level 5 */ + { 21, 16, 19, 2, 5, 16, ZSTD_lazy }, /* level 6 */ + { 21, 17, 20, 3, 5, 16, ZSTD_lazy }, /* level 7 */ + { 21, 18, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 8 */ + { 21, 20, 20, 3, 5, 16, ZSTD_lazy2 }, /* level 9 */ + { 21, 19, 21, 4, 5, 16, ZSTD_lazy2 }, /* level 10 */ + { 22, 20, 22, 4, 5, 16, ZSTD_lazy2 }, /* level 11 */ + { 22, 20, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 12 */ + { 22, 21, 22, 5, 5, 16, ZSTD_lazy2 }, /* level 13 */ + { 22, 21, 22, 6, 5, 16, ZSTD_lazy2 }, /* level 14 */ + { 22, 21, 21, 5, 5, 16, ZSTD_btlazy2 }, /* level 15 */ + { 23, 22, 22, 5, 5, 16, ZSTD_btlazy2 }, /* level 16 */ + { 23, 21, 22, 4, 5, 24, ZSTD_btopt }, /* level 17 */ + { 23, 23, 22, 6, 5, 32, ZSTD_btopt }, /* level 18 */ + { 23, 23, 22, 6, 3, 48, ZSTD_btopt }, /* level 19 */ + { 25, 25, 23, 7, 3, 64, ZSTD_btopt2 }, /* level 20 */ + { 26, 26, 23, 7, 3,256, ZSTD_btopt2 }, /* level 21 */ + { 27, 27, 25, 9, 3,512, ZSTD_btopt2 }, /* level 22 */ +}, +{ /* for srcSize <= 256 KB */ + /* W, C, H, S, L, T, strat */ + { 0, 0, 0, 0, 0, 0, ZSTD_fast }, /* level 0 - not used */ + { 18, 13, 14, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 18, 14, 13, 1, 5, 8, ZSTD_dfast }, /* level 2 */ + { 18, 16, 15, 1, 5, 8, ZSTD_dfast }, /* level 3 */ + { 18, 15, 17, 1, 5, 8, ZSTD_greedy }, /* level 4.*/ + { 18, 16, 17, 4, 5, 8, ZSTD_greedy }, /* level 5.*/ + { 18, 16, 17, 3, 5, 8, ZSTD_lazy }, /* level 6.*/ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy }, /* level 7 */ + { 18, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 18, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 18, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 18, 18, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 11.*/ + { 18, 18, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 12.*/ + { 18, 19, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13 */ + { 18, 18, 18, 4, 4, 16, ZSTD_btopt }, /* level 14.*/ + { 18, 18, 18, 4, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 18, 19, 18, 6, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 18, 19, 18, 8, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 18, 19, 18, 9, 3,128, ZSTD_btopt }, /* level 18.*/ + { 18, 19, 18, 10, 3,256, ZSTD_btopt }, /* level 19.*/ + { 18, 19, 18, 11, 3,512, ZSTD_btopt2 }, /* level 20.*/ + { 18, 19, 18, 12, 3,512, ZSTD_btopt2 }, /* level 21.*/ + { 18, 19, 18, 13, 3,512, ZSTD_btopt2 }, /* level 22.*/ +}, +{ /* for srcSize <= 128 KB */ + /* W, C, H, S, L, T, strat */ + { 17, 12, 12, 1, 7, 8, ZSTD_fast }, /* level 0 - not used */ + { 17, 12, 13, 1, 6, 8, ZSTD_fast }, /* level 1 */ + { 17, 13, 16, 1, 5, 8, ZSTD_fast }, /* level 2 */ + { 17, 16, 16, 2, 5, 8, ZSTD_dfast }, /* level 3 */ + { 17, 13, 15, 3, 4, 8, ZSTD_greedy }, /* level 4 */ + { 17, 15, 17, 4, 4, 8, ZSTD_greedy }, /* level 5 */ + { 17, 16, 17, 3, 4, 8, ZSTD_lazy }, /* level 6 */ + { 17, 15, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 7 */ + { 17, 17, 17, 4, 4, 8, ZSTD_lazy2 }, /* level 8 */ + { 17, 17, 17, 5, 4, 8, ZSTD_lazy2 }, /* level 9 */ + { 17, 17, 17, 6, 4, 8, ZSTD_lazy2 }, /* level 10 */ + { 17, 17, 17, 7, 4, 8, ZSTD_lazy2 }, /* level 11 */ + { 17, 17, 17, 8, 4, 8, ZSTD_lazy2 }, /* level 12 */ + { 17, 18, 17, 6, 4, 8, ZSTD_btlazy2 }, /* level 13.*/ + { 17, 17, 17, 7, 3, 8, ZSTD_btopt }, /* level 14.*/ + { 17, 17, 17, 7, 3, 16, ZSTD_btopt }, /* level 15.*/ + { 17, 18, 17, 7, 3, 32, ZSTD_btopt }, /* level 16.*/ + { 17, 18, 17, 7, 3, 64, ZSTD_btopt }, /* level 17.*/ + { 17, 18, 17, 7, 3,256, ZSTD_btopt }, /* level 18.*/ + { 17, 18, 17, 8, 3,256, ZSTD_btopt }, /* level 19.*/ + { 17, 18, 17, 9, 3,256, ZSTD_btopt2 }, /* level 20.*/ + { 17, 18, 17, 10, 3,256, ZSTD_btopt2 }, /* level 21.*/ + { 17, 18, 17, 11, 3,512, ZSTD_btopt2 }, /* level 22.*/ +}, +{ /* for srcSize <= 16 KB */ + /* W, C, H, S, L, T, strat */ + { 14, 12, 12, 1, 7, 6, ZSTD_fast }, /* level 0 - not used */ + { 14, 14, 14, 1, 6, 6, ZSTD_fast }, /* level 1 */ + { 14, 14, 14, 1, 4, 6, ZSTD_fast }, /* level 2 */ + { 14, 14, 14, 1, 4, 6, ZSTD_dfast }, /* level 3.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_greedy }, /* level 4.*/ + { 14, 14, 14, 3, 4, 6, ZSTD_lazy }, /* level 5.*/ + { 14, 14, 14, 4, 4, 6, ZSTD_lazy2 }, /* level 6 */ + { 14, 14, 14, 5, 4, 6, ZSTD_lazy2 }, /* level 7 */ + { 14, 14, 14, 6, 4, 6, ZSTD_lazy2 }, /* level 8.*/ + { 14, 15, 14, 6, 4, 6, ZSTD_btlazy2 }, /* level 9.*/ + { 14, 15, 14, 3, 3, 6, ZSTD_btopt }, /* level 10.*/ + { 14, 15, 14, 6, 3, 8, ZSTD_btopt }, /* level 11.*/ + { 14, 15, 14, 6, 3, 16, ZSTD_btopt }, /* level 12.*/ + { 14, 15, 14, 6, 3, 24, ZSTD_btopt }, /* level 13.*/ + { 14, 15, 15, 6, 3, 48, ZSTD_btopt }, /* level 14.*/ + { 14, 15, 15, 6, 3, 64, ZSTD_btopt }, /* level 15.*/ + { 14, 15, 15, 6, 3, 96, ZSTD_btopt }, /* level 16.*/ + { 14, 15, 15, 6, 3,128, ZSTD_btopt }, /* level 17.*/ + { 14, 15, 15, 6, 3,256, ZSTD_btopt }, /* level 18.*/ + { 14, 15, 15, 7, 3,256, ZSTD_btopt }, /* level 19.*/ + { 14, 15, 15, 8, 3,256, ZSTD_btopt2 }, /* level 20.*/ + { 14, 15, 15, 9, 3,256, ZSTD_btopt2 }, /* level 21.*/ + { 14, 15, 15, 10, 3,256, ZSTD_btopt2 }, /* level 22.*/ +}, +}; + +/*! ZSTD_getCParams() : +* @return ZSTD_compressionParameters structure for a selected compression level, `srcSize` and `dictSize`. +* Size values are optional, provide 0 if not known or unused */ +ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) +{ + ZSTD_compressionParameters cp; + size_t const addedSize = srcSize ? 0 : 500; + U64 const rSize = srcSize+dictSize ? srcSize+dictSize+addedSize : (U64)-1; + U32 const tableID = (rSize <= 256 KB) + (rSize <= 128 KB) + (rSize <= 16 KB); /* intentional underflow for srcSizeHint == 0 */ + if (compressionLevel <= 0) compressionLevel = ZSTD_DEFAULT_CLEVEL; /* 0 == default; no negative compressionLevel yet */ + if (compressionLevel > ZSTD_MAX_CLEVEL) compressionLevel = ZSTD_MAX_CLEVEL; + cp = ZSTD_defaultCParameters[tableID][compressionLevel]; + if (MEM_32bits()) { /* auto-correction, for 32-bits mode */ + if (cp.windowLog > ZSTD_WINDOWLOG_MAX) cp.windowLog = ZSTD_WINDOWLOG_MAX; + if (cp.chainLog > ZSTD_CHAINLOG_MAX) cp.chainLog = ZSTD_CHAINLOG_MAX; + if (cp.hashLog > ZSTD_HASHLOG_MAX) cp.hashLog = ZSTD_HASHLOG_MAX; + } + cp = ZSTD_adjustCParams(cp, srcSize, dictSize); + return cp; +} + +/*! ZSTD_getParams() : +* same as ZSTD_getCParams(), but @return a `ZSTD_parameters` object (instead of `ZSTD_compressionParameters`). +* All fields of `ZSTD_frameParameters` are set to default (0) */ +ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSize, size_t dictSize) { + ZSTD_parameters params; + ZSTD_compressionParameters const cParams = ZSTD_getCParams(compressionLevel, srcSize, dictSize); + memset(¶ms, 0, sizeof(params)); + params.cParams = cParams; + return params; +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/zstd_opt.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/zstd_opt.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/compress/zstd_opt.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/compress/zstd_opt.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,919 @@ +/** + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/* Note : this file is intended to be included within zstd_compress.c */ + + +#ifndef ZSTD_OPT_H_91842398743 +#define ZSTD_OPT_H_91842398743 + + +#define ZSTD_LITFREQ_ADD 2 +#define ZSTD_FREQ_DIV 4 +#define ZSTD_MAX_PRICE (1<<30) + +/*-************************************* +* Price functions for optimal parser +***************************************/ +FORCE_INLINE void ZSTD_setLog2Prices(seqStore_t* ssPtr) +{ + ssPtr->log2matchLengthSum = ZSTD_highbit32(ssPtr->matchLengthSum+1); + ssPtr->log2litLengthSum = ZSTD_highbit32(ssPtr->litLengthSum+1); + ssPtr->log2litSum = ZSTD_highbit32(ssPtr->litSum+1); + ssPtr->log2offCodeSum = ZSTD_highbit32(ssPtr->offCodeSum+1); + ssPtr->factor = 1 + ((ssPtr->litSum>>5) / ssPtr->litLengthSum) + ((ssPtr->litSum<<1) / (ssPtr->litSum + ssPtr->matchSum)); +} + + +MEM_STATIC void ZSTD_rescaleFreqs(seqStore_t* ssPtr, const BYTE* src, size_t srcSize) +{ + unsigned u; + + ssPtr->cachedLiterals = NULL; + ssPtr->cachedPrice = ssPtr->cachedLitLength = 0; + ssPtr->staticPrices = 0; + + if (ssPtr->litLengthSum == 0) { + if (srcSize <= 1024) ssPtr->staticPrices = 1; + + for (u=0; u<=MaxLit; u++) + ssPtr->litFreq[u] = 0; + for (u=0; ulitFreq[src[u]]++; + + ssPtr->litSum = 0; + ssPtr->litLengthSum = MaxLL+1; + ssPtr->matchLengthSum = MaxML+1; + ssPtr->offCodeSum = (MaxOff+1); + ssPtr->matchSum = (ZSTD_LITFREQ_ADD<litFreq[u] = 1 + (ssPtr->litFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->litSum += ssPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) + ssPtr->litLengthFreq[u] = 1; + for (u=0; u<=MaxML; u++) + ssPtr->matchLengthFreq[u] = 1; + for (u=0; u<=MaxOff; u++) + ssPtr->offCodeFreq[u] = 1; + } else { + ssPtr->matchLengthSum = 0; + ssPtr->litLengthSum = 0; + ssPtr->offCodeSum = 0; + ssPtr->matchSum = 0; + ssPtr->litSum = 0; + + for (u=0; u<=MaxLit; u++) { + ssPtr->litFreq[u] = 1 + (ssPtr->litFreq[u]>>(ZSTD_FREQ_DIV+1)); + ssPtr->litSum += ssPtr->litFreq[u]; + } + for (u=0; u<=MaxLL; u++) { + ssPtr->litLengthFreq[u] = 1 + (ssPtr->litLengthFreq[u]>>(ZSTD_FREQ_DIV+1)); + ssPtr->litLengthSum += ssPtr->litLengthFreq[u]; + } + for (u=0; u<=MaxML; u++) { + ssPtr->matchLengthFreq[u] = 1 + (ssPtr->matchLengthFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->matchLengthSum += ssPtr->matchLengthFreq[u]; + ssPtr->matchSum += ssPtr->matchLengthFreq[u] * (u + 3); + } + ssPtr->matchSum *= ZSTD_LITFREQ_ADD; + for (u=0; u<=MaxOff; u++) { + ssPtr->offCodeFreq[u] = 1 + (ssPtr->offCodeFreq[u]>>ZSTD_FREQ_DIV); + ssPtr->offCodeSum += ssPtr->offCodeFreq[u]; + } + } + + ZSTD_setLog2Prices(ssPtr); +} + + +FORCE_INLINE U32 ZSTD_getLiteralPrice(seqStore_t* ssPtr, U32 litLength, const BYTE* literals) +{ + U32 price, u; + + if (ssPtr->staticPrices) + return ZSTD_highbit32((U32)litLength+1) + (litLength*6); + + if (litLength == 0) + return ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[0]+1); + + /* literals */ + if (ssPtr->cachedLiterals == literals) { + U32 const additional = litLength - ssPtr->cachedLitLength; + const BYTE* literals2 = ssPtr->cachedLiterals + ssPtr->cachedLitLength; + price = ssPtr->cachedPrice + additional * ssPtr->log2litSum; + for (u=0; u < additional; u++) + price -= ZSTD_highbit32(ssPtr->litFreq[literals2[u]]+1); + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } else { + price = litLength * ssPtr->log2litSum; + for (u=0; u < litLength; u++) + price -= ZSTD_highbit32(ssPtr->litFreq[literals[u]]+1); + + if (litLength >= 12) { + ssPtr->cachedLiterals = literals; + ssPtr->cachedPrice = price; + ssPtr->cachedLitLength = litLength; + } + } + + /* literal Length */ + { const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + price += LL_bits[llCode] + ssPtr->log2litLengthSum - ZSTD_highbit32(ssPtr->litLengthFreq[llCode]+1); + } + + return price; +} + + +FORCE_INLINE U32 ZSTD_getPrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength, const int ultra) +{ + /* offset */ + U32 price; + BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + + if (seqStorePtr->staticPrices) + return ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + ZSTD_highbit32((U32)matchLength+1) + 16 + offCode; + + price = offCode + seqStorePtr->log2offCodeSum - ZSTD_highbit32(seqStorePtr->offCodeFreq[offCode]+1); + if (!ultra && offCode >= 20) price += (offCode-19)*2; + + /* match Length */ + { const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + price += ML_bits[mlCode] + seqStorePtr->log2matchLengthSum - ZSTD_highbit32(seqStorePtr->matchLengthFreq[mlCode]+1); + } + + return price + ZSTD_getLiteralPrice(seqStorePtr, litLength, literals) + seqStorePtr->factor; +} + + +MEM_STATIC void ZSTD_updatePrice(seqStore_t* seqStorePtr, U32 litLength, const BYTE* literals, U32 offset, U32 matchLength) +{ + U32 u; + + /* literals */ + seqStorePtr->litSum += litLength*ZSTD_LITFREQ_ADD; + for (u=0; u < litLength; u++) + seqStorePtr->litFreq[literals[u]] += ZSTD_LITFREQ_ADD; + + /* literal Length */ + { const BYTE LL_deltaCode = 19; + const BYTE llCode = (litLength>63) ? (BYTE)ZSTD_highbit32(litLength) + LL_deltaCode : LL_Code[litLength]; + seqStorePtr->litLengthFreq[llCode]++; + seqStorePtr->litLengthSum++; + } + + /* match offset */ + { BYTE const offCode = (BYTE)ZSTD_highbit32(offset+1); + seqStorePtr->offCodeSum++; + seqStorePtr->offCodeFreq[offCode]++; + } + + /* match Length */ + { const BYTE ML_deltaCode = 36; + const BYTE mlCode = (matchLength>127) ? (BYTE)ZSTD_highbit32(matchLength) + ML_deltaCode : ML_Code[matchLength]; + seqStorePtr->matchLengthFreq[mlCode]++; + seqStorePtr->matchLengthSum++; + } + + ZSTD_setLog2Prices(seqStorePtr); +} + + +#define SET_PRICE(pos, mlen_, offset_, litlen_, price_) \ + { \ + while (last_pos < pos) { opt[last_pos+1].price = ZSTD_MAX_PRICE; last_pos++; } \ + opt[pos].mlen = mlen_; \ + opt[pos].off = offset_; \ + opt[pos].litlen = litlen_; \ + opt[pos].price = price_; \ + } + + + +/* Update hashTable3 up to ip (excluded) + Assumption : always within prefix (ie. not within extDict) */ +FORCE_INLINE +U32 ZSTD_insertAndFindFirstIndexHash3 (ZSTD_CCtx* zc, const BYTE* ip) +{ + U32* const hashTable3 = zc->hashTable3; + U32 const hashLog3 = zc->hashLog3; + const BYTE* const base = zc->base; + U32 idx = zc->nextToUpdate3; + const U32 target = zc->nextToUpdate3 = (U32)(ip - base); + const size_t hash3 = ZSTD_hash3Ptr(ip, hashLog3); + + while(idx < target) { + hashTable3[ZSTD_hash3Ptr(base+idx, hashLog3)] = idx; + idx++; + } + + return hashTable3[hash3]; +} + + +/*-************************************* +* Binary Tree search +***************************************/ +static U32 ZSTD_insertBtAndGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + U32 nbCompares, const U32 mls, + U32 extDict, ZSTD_match_t* matches, const U32 minMatchLen) +{ + const BYTE* const base = zc->base; + const U32 current = (U32)(ip-base); + const U32 hashLog = zc->params.cParams.hashLog; + const size_t h = ZSTD_hashPtr(ip, hashLog, mls); + U32* const hashTable = zc->hashTable; + U32 matchIndex = hashTable[h]; + U32* const bt = zc->chainTable; + const U32 btLog = zc->params.cParams.chainLog - 1; + const U32 btMask= (1U << btLog) - 1; + size_t commonLengthSmaller=0, commonLengthLarger=0; + const BYTE* const dictBase = zc->dictBase; + const U32 dictLimit = zc->dictLimit; + const BYTE* const dictEnd = dictBase + dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const U32 btLow = btMask >= current ? 0 : current - btMask; + const U32 windowLow = zc->lowLimit; + U32* smallerPtr = bt + 2*(current&btMask); + U32* largerPtr = bt + 2*(current&btMask) + 1; + U32 matchEndIdx = current+8; + U32 dummy32; /* to be nullified at the end */ + U32 mnum = 0; + + const U32 minMatch = (mls == 3) ? 3 : 4; + size_t bestLength = minMatchLen-1; + + if (minMatch == 3) { /* HC3 match finder */ + U32 const matchIndex3 = ZSTD_insertAndFindFirstIndexHash3 (zc, ip); + if (matchIndex3>windowLow && (current - matchIndex3 < (1<<18))) { + const BYTE* match; + size_t currentMl=0; + if ((!extDict) || matchIndex3 >= dictLimit) { + match = base + matchIndex3; + if (match[bestLength] == ip[bestLength]) currentMl = ZSTD_count(ip, match, iLimit); + } else { + match = dictBase + matchIndex3; + if (MEM_readMINMATCH(match, MINMATCH) == MEM_readMINMATCH(ip, MINMATCH)) /* assumption : matchIndex3 <= dictLimit-4 (by table construction) */ + currentMl = ZSTD_count_2segments(ip+MINMATCH, match+MINMATCH, iLimit, dictEnd, prefixStart) + MINMATCH; + } + + /* save best solution */ + if (currentMl > bestLength) { + bestLength = currentMl; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex3; + matches[mnum].len = (U32)currentMl; + mnum++; + if (currentMl > ZSTD_OPT_NUM) goto update; + if (ip+currentMl == iLimit) goto update; /* best possible, and avoid read overflow*/ + } + } + } + + hashTable[h] = current; /* Update Hash Table */ + + while (nbCompares-- && (matchIndex > windowLow)) { + U32* nextPtr = bt + 2*(matchIndex & btMask); + size_t matchLength = MIN(commonLengthSmaller, commonLengthLarger); /* guaranteed minimum nb of common bytes */ + const BYTE* match; + + if ((!extDict) || (matchIndex+matchLength >= dictLimit)) { + match = base + matchIndex; + if (match[matchLength] == ip[matchLength]) { + matchLength += ZSTD_count(ip+matchLength+1, match+matchLength+1, iLimit) +1; + } + } else { + match = dictBase + matchIndex; + matchLength += ZSTD_count_2segments(ip+matchLength, match+matchLength, iLimit, dictEnd, prefixStart); + if (matchIndex+matchLength >= dictLimit) + match = base + matchIndex; /* to prepare for next usage of match[matchLength] */ + } + + if (matchLength > bestLength) { + if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; + bestLength = matchLength; + matches[mnum].off = ZSTD_REP_MOVE_OPT + current - matchIndex; + matches[mnum].len = (U32)matchLength; + mnum++; + if (matchLength > ZSTD_OPT_NUM) break; + if (ip+matchLength == iLimit) /* equal : no way to know if inf or sup */ + break; /* drop, to guarantee consistency (miss a little bit of compression) */ + } + + if (match[matchLength] < ip[matchLength]) { + /* match is smaller than current */ + *smallerPtr = matchIndex; /* update smaller idx */ + commonLengthSmaller = matchLength; /* all smaller will now have at least this guaranteed common length */ + if (matchIndex <= btLow) { smallerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + smallerPtr = nextPtr+1; /* new "smaller" => larger of match */ + matchIndex = nextPtr[1]; /* new matchIndex larger than previous (closer to current) */ + } else { + /* match is larger than current */ + *largerPtr = matchIndex; + commonLengthLarger = matchLength; + if (matchIndex <= btLow) { largerPtr=&dummy32; break; } /* beyond tree size, stop the search */ + largerPtr = nextPtr; + matchIndex = nextPtr[0]; + } } + + *smallerPtr = *largerPtr = 0; + +update: + zc->nextToUpdate = (matchEndIdx > current + 8) ? matchEndIdx - 8 : current+1; + return mnum; +} + + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 0, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + +/** Tree updater, providing best match */ +static U32 ZSTD_BtGetAllMatches_extDict ( + ZSTD_CCtx* zc, + const BYTE* const ip, const BYTE* const iLimit, + const U32 maxNbAttempts, const U32 mls, ZSTD_match_t* matches, const U32 minMatchLen) +{ + if (ip < zc->base + zc->nextToUpdate) return 0; /* skipped area */ + ZSTD_updateTree_extDict(zc, ip, iLimit, maxNbAttempts, mls); + return ZSTD_insertBtAndGetAllMatches(zc, ip, iLimit, maxNbAttempts, mls, 1, matches, minMatchLen); +} + + +static U32 ZSTD_BtGetAllMatches_selectMLS_extDict ( + ZSTD_CCtx* zc, /* Index table will be updated */ + const BYTE* ip, const BYTE* const iHighLimit, + const U32 maxNbAttempts, const U32 matchLengthSearch, ZSTD_match_t* matches, const U32 minMatchLen) +{ + switch(matchLengthSearch) + { + case 3 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 3, matches, minMatchLen); + default : + case 4 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 4, matches, minMatchLen); + case 5 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 5, matches, minMatchLen); + case 6 : return ZSTD_BtGetAllMatches_extDict(zc, ip, iHighLimit, maxNbAttempts, 6, matches, minMatchLen); + } +} + + +/*-******************************* +* Optimal parser +*********************************/ +FORCE_INLINE +void ZSTD_compressBlock_opt_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, const int ultra) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const BYTE* const prefixStart = base + ctx->dictLimit; + + const U32 maxSearches = 1U << ctx->params.cParams.searchLog; + const U32 sufficient_len = ctx->params.cParams.targetLength; + const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = seqStorePtr->priceTable; + ZSTD_match_t* matches = seqStorePtr->matchTable; + const BYTE* inr; + U32 offset, rep[ZSTD_REP_NUM]; + + /* init */ + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize); + ip += (ip==prefixStart); + { U32 i; for (i=0; irep[i]; } + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); + for (i=(ip == anchor); i 0) && (repCur < (S32)(ip-prefixStart)) + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(ip - repCur, minMatch))) { + mlen = (U32)ZSTD_count(ip+minMatch, ip+minMatch-repCur, iend) + minMatch; + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + best_off = i - (ip == anchor); + do { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, ip, iend, maxSearches, mls, matches, minMatch); + + if (!last_pos && !match_num) { ip++; continue; } + + if (match_num && (matches[match_num-1].len > sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + /* set prices using matches at position = 0 */ + best_mlen = (last_pos) ? last_pos : minMatch; + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); /* note : macro modifies last_pos */ + mlen++; + } } + + if (last_pos < minMatch) { ip++; continue; } + + /* initialize opt[0] */ + { U32 i ; for (i=0; i litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + } + + best_mlen = minMatch; + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); + for (i=(opt[cur].mlen != 1); i 0) && (repCur < (S32)(inr-prefixStart)) + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(inr - repCur, minMatch))) { + mlen = (U32)ZSTD_count(inr+minMatch, inr+minMatch - repCur, iend) + minMatch; + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = i - (opt[cur].mlen != 1); + if (mlen > best_mlen) best_mlen = mlen; + + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS(ctx, inr, iend, maxSearches, mls, matches, best_mlen); + + if (match_num > 0 && (matches[match_num-1].len > sufficient_len || cur + matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); + else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos;) { + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + + if (offset > ZSTD_REP_MOVE_OPT) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; + } else { + if (offset != 0) { + best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + if (litLength==0) offset--; + } + + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + /* Save reps for next block */ + { int i; for (i=0; isavedRep[i] = rep[i]; } + + /* Last Literals */ + { size_t const lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + + +FORCE_INLINE +void ZSTD_compressBlock_opt_extDict_generic(ZSTD_CCtx* ctx, + const void* src, size_t srcSize, const int ultra) +{ + seqStore_t* seqStorePtr = &(ctx->seqStore); + const BYTE* const istart = (const BYTE*)src; + const BYTE* ip = istart; + const BYTE* anchor = istart; + const BYTE* const iend = istart + srcSize; + const BYTE* const ilimit = iend - 8; + const BYTE* const base = ctx->base; + const U32 lowestIndex = ctx->lowLimit; + const U32 dictLimit = ctx->dictLimit; + const BYTE* const prefixStart = base + dictLimit; + const BYTE* const dictBase = ctx->dictBase; + const BYTE* const dictEnd = dictBase + dictLimit; + + const U32 maxSearches = 1U << ctx->params.cParams.searchLog; + const U32 sufficient_len = ctx->params.cParams.targetLength; + const U32 mls = ctx->params.cParams.searchLength; + const U32 minMatch = (ctx->params.cParams.searchLength == 3) ? 3 : 4; + + ZSTD_optimal_t* opt = seqStorePtr->priceTable; + ZSTD_match_t* matches = seqStorePtr->matchTable; + const BYTE* inr; + + /* init */ + U32 offset, rep[ZSTD_REP_NUM]; + { U32 i; for (i=0; irep[i]; } + + ctx->nextToUpdate3 = ctx->nextToUpdate; + ZSTD_rescaleFreqs(seqStorePtr, (const BYTE*)src, srcSize); + ip += (ip==prefixStart); + + /* Match Loop */ + while (ip < ilimit) { + U32 cur, match_num, last_pos, litlen, price; + U32 u, mlen, best_mlen, best_off, litLength; + U32 current = (U32)(ip-base); + memset(opt, 0, sizeof(ZSTD_optimal_t)); + last_pos = 0; + opt[0].litlen = (U32)(ip - anchor); + + /* check repCode */ + { U32 i, last_i = ZSTD_REP_CHECK + (ip==anchor); + for (i = (ip==anchor); i 0 && repCur <= (S32)current) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + && (MEM_readMINMATCH(ip, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected we should take it */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(ip+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + if (mlen > sufficient_len || mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; cur = 0; last_pos = 1; + goto _storeSequence; + } + + best_off = i - (ip==anchor); + litlen = opt[0].litlen; + do { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, i, litlen, price); /* note : macro modifies last_pos */ + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, ip, iend, maxSearches, mls, matches, minMatch); /* first search (depth 0) */ + + if (!last_pos && !match_num) { ip++; continue; } + + { U32 i; for (i=0; i sufficient_len || matches[match_num-1].len >= ZSTD_OPT_NUM)) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + cur = 0; + last_pos = 1; + goto _storeSequence; + } + + best_mlen = (last_pos) ? last_pos : minMatch; + + /* set prices using matches at position = 0 */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = matches[u].len; + litlen = opt[0].litlen; + while (mlen <= best_mlen) { + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + if (mlen > last_pos || price < opt[mlen].price) + SET_PRICE(mlen, mlen, matches[u].off, litlen, price); + mlen++; + } } + + if (last_pos < minMatch) { + ip++; continue; + } + + /* check further positions */ + for (cur = 1; cur <= last_pos; cur++) { + inr = ip + cur; + + if (opt[cur-1].mlen == 1) { + litlen = opt[cur-1].litlen + 1; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-litlen); + } else + price = ZSTD_getLiteralPrice(seqStorePtr, litlen, anchor); + } else { + litlen = 1; + price = opt[cur - 1].price + ZSTD_getLiteralPrice(seqStorePtr, litlen, inr-1); + } + + if (cur > last_pos || price <= opt[cur].price) + SET_PRICE(cur, 1, 0, litlen, price); + + if (cur == last_pos) break; + + if (inr > ilimit) /* last match must start at a minimum distance of 8 from oend */ + continue; + + mlen = opt[cur].mlen; + if (opt[cur].off > ZSTD_REP_MOVE_OPT) { + opt[cur].rep[2] = opt[cur-mlen].rep[1]; + opt[cur].rep[1] = opt[cur-mlen].rep[0]; + opt[cur].rep[0] = opt[cur].off - ZSTD_REP_MOVE_OPT; + } else { + opt[cur].rep[2] = (opt[cur].off > 1) ? opt[cur-mlen].rep[1] : opt[cur-mlen].rep[2]; + opt[cur].rep[1] = (opt[cur].off > 0) ? opt[cur-mlen].rep[0] : opt[cur-mlen].rep[1]; + opt[cur].rep[0] = ((opt[cur].off==ZSTD_REP_MOVE_OPT) && (mlen != 1)) ? (opt[cur-mlen].rep[0] - 1) : (opt[cur-mlen].rep[opt[cur].off]); + } + + best_mlen = minMatch; + { U32 i, last_i = ZSTD_REP_CHECK + (mlen != 1); + for (i = (mlen != 1); i 0 && repCur <= (S32)(current+cur)) + && (((U32)((dictLimit-1) - repIndex) >= 3) & (repIndex>lowestIndex)) /* intentional overflow */ + && (MEM_readMINMATCH(inr, minMatch) == MEM_readMINMATCH(repMatch, minMatch)) ) { + /* repcode detected */ + const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; + mlen = (U32)ZSTD_count_2segments(inr+minMatch, repMatch+minMatch, iend, repEnd, prefixStart) + minMatch; + + if (mlen > sufficient_len || cur + mlen >= ZSTD_OPT_NUM) { + best_mlen = mlen; best_off = i; last_pos = cur + 1; + goto _storeSequence; + } + + best_off = i - (opt[cur].mlen != 1); + if (mlen > best_mlen) best_mlen = mlen; + + do { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) { + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, inr-litlen, best_off, mlen - MINMATCH, ultra); + } else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, best_off, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, best_off, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || price <= opt[cur + mlen].price) + SET_PRICE(cur + mlen, mlen, i, litlen, price); + mlen--; + } while (mlen >= minMatch); + } } } + + match_num = ZSTD_BtGetAllMatches_selectMLS_extDict(ctx, inr, iend, maxSearches, mls, matches, minMatch); + + if (match_num > 0 && matches[match_num-1].len > sufficient_len) { + best_mlen = matches[match_num-1].len; + best_off = matches[match_num-1].off; + last_pos = cur + 1; + goto _storeSequence; + } + + /* set prices using matches at position = cur */ + for (u = 0; u < match_num; u++) { + mlen = (u>0) ? matches[u-1].len+1 : best_mlen; + best_mlen = (cur + matches[u].len < ZSTD_OPT_NUM) ? matches[u].len : ZSTD_OPT_NUM - cur; + + while (mlen <= best_mlen) { + if (opt[cur].mlen == 1) { + litlen = opt[cur].litlen; + if (cur > litlen) + price = opt[cur - litlen].price + ZSTD_getPrice(seqStorePtr, litlen, ip+cur-litlen, matches[u].off-1, mlen - MINMATCH, ultra); + else + price = ZSTD_getPrice(seqStorePtr, litlen, anchor, matches[u].off-1, mlen - MINMATCH, ultra); + } else { + litlen = 0; + price = opt[cur].price + ZSTD_getPrice(seqStorePtr, 0, NULL, matches[u].off-1, mlen - MINMATCH, ultra); + } + + if (cur + mlen > last_pos || (price < opt[cur + mlen].price)) + SET_PRICE(cur + mlen, mlen, matches[u].off, litlen, price); + + mlen++; + } } } /* for (cur = 1; cur <= last_pos; cur++) */ + + best_mlen = opt[last_pos].mlen; + best_off = opt[last_pos].off; + cur = last_pos - best_mlen; + + /* store sequence */ +_storeSequence: /* cur, last_pos, best_mlen, best_off have to be set */ + opt[0].mlen = 1; + + while (1) { + mlen = opt[cur].mlen; + offset = opt[cur].off; + opt[cur].mlen = best_mlen; + opt[cur].off = best_off; + best_mlen = mlen; + best_off = offset; + if (mlen > cur) break; + cur -= mlen; + } + + for (u = 0; u <= last_pos; ) { + u += opt[u].mlen; + } + + for (cur=0; cur < last_pos; ) { + mlen = opt[cur].mlen; + if (mlen == 1) { ip++; cur++; continue; } + offset = opt[cur].off; + cur += mlen; + litLength = (U32)(ip - anchor); + + if (offset > ZSTD_REP_MOVE_OPT) { + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = offset - ZSTD_REP_MOVE_OPT; + offset--; + } else { + if (offset != 0) { + best_off = (offset==ZSTD_REP_MOVE_OPT) ? (rep[0] - 1) : (rep[offset]); + if (offset != 1) rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = best_off; + } + + if (litLength==0) offset--; + } + + ZSTD_updatePrice(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + ZSTD_storeSeq(seqStorePtr, litLength, anchor, offset, mlen-MINMATCH); + anchor = ip = ip + mlen; + } } /* for (cur=0; cur < last_pos; ) */ + + /* Save reps for next block */ + { int i; for (i=0; isavedRep[i] = rep[i]; } + + /* Last Literals */ + { size_t lastLLSize = iend - anchor; + memcpy(seqStorePtr->lit, anchor, lastLLSize); + seqStorePtr->lit += lastLLSize; + } +} + +#endif /* ZSTD_OPT_H_91842398743 */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/decompress/huf_decompress.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/decompress/huf_decompress.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/decompress/huf_decompress.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/decompress/huf_decompress.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,885 @@ +/* ****************************************************************** + Huffman decoder, part of New Generation Entropy library + Copyright (C) 2013-2016, Yann Collet. + + BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + You can contact the author at : + - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy + - Public forum : https://groups.google.com/forum/#!forum/lz4c +****************************************************************** */ + +/* ************************************************************** +* Compiler specifics +****************************************************************/ +#if defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) +/* inline is defined */ +#elif defined(_MSC_VER) || defined(__GNUC__) +# define inline __inline +#else +# define inline /* disable inline */ +#endif + +#ifdef _MSC_VER /* Visual Studio */ +# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */ +#endif + + +/* ************************************************************** +* Dependencies +****************************************************************/ +#include /* memcpy, memset */ +#include "bitstream.h" /* BIT_* */ +#include "fse.h" /* header compression */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" + + +/* ************************************************************** +* Error Management +****************************************************************/ +#define HUF_STATIC_ASSERT(c) { enum { HUF_static_assert = 1/(int)(!!(c)) }; } /* use only *after* variable declarations */ + + +/*-***************************/ +/* generic DTableDesc */ +/*-***************************/ + +typedef struct { BYTE maxTableLog; BYTE tableType; BYTE tableLog; BYTE reserved; } DTableDesc; + +static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) +{ + DTableDesc dtd; + memcpy(&dtd, table, sizeof(dtd)); + return dtd; +} + + +/*-***************************/ +/* single-symbol decoding */ +/*-***************************/ + +typedef struct { BYTE byte; BYTE nbBits; } HUF_DEltX2; /* single-symbol decoding */ + +size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankVal[HUF_TABLELOG_ABSOLUTEMAX + 1]; /* large enough for values from 0 to 16 */ + U32 tableLog = 0; + U32 nbSymbols = 0; + size_t iSize; + void* const dtPtr = DTable + 1; + HUF_DEltX2* const dt = (HUF_DEltX2*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); + /* memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(huffWeight, HUF_SYMBOLVALUE_MAX + 1, rankVal, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* Table header */ + { DTableDesc dtd = HUF_getDTableDesc(DTable); + if (tableLog > (U32)(dtd.maxTableLog+1)) return ERROR(tableLog_tooLarge); /* DTable too small, huffman tree cannot fit in */ + dtd.tableType = 0; + dtd.tableLog = (BYTE)tableLog; + memcpy(DTable, &dtd, sizeof(dtd)); + } + + /* Prepare ranks */ + { U32 n, nextRankStart = 0; + for (n=1; n> 1; + U32 i; + HUF_DEltX2 D; + D.byte = (BYTE)n; D.nbBits = (BYTE)(tableLog + 1 - w); + for (i = rankVal[w]; i < rankVal[w] + length; i++) + dt[i] = D; + rankVal[w] += length; + } } + + return iSize; +} + + +static BYTE HUF_decodeSymbolX2(BIT_DStream_t* Dstream, const HUF_DEltX2* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(Dstream, dtLog); /* note : dtLog >= 1 */ + BYTE const c = dt[val].byte; + BIT_skipBits(Dstream, dt[val].nbBits); + return c; +} + +#define HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) \ + *ptr++ = HUF_decodeSymbolX2(DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX2_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +#define HUF_DECODE_SYMBOLX2_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + HUF_DECODE_SYMBOLX2_0(ptr, DStreamPtr) + +static inline size_t HUF_decodeStreamX2(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, const HUF_DEltX2* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 4 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p <= pEnd-4)) { + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_1(p, bitDPtr); + HUF_DECODE_SYMBOLX2_2(p, bitDPtr); + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + } + + /* closer to the end */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) && (p < pEnd)) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + /* no more data to retrieve from bitstream, hence no need to reload */ + while (p < pEnd) + HUF_DECODE_SYMBOLX2_0(p, bitDPtr); + + return pEnd-pStart; +} + +static size_t HUF_decompress1X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BYTE* op = (BYTE*)dst; + BYTE* const oend = op + dstSize; + const void* dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + BIT_DStream_t bitD; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; } + + HUF_decodeStreamX2(op, &bitD, oend, dt, dtLog); + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + return dstSize; +} + +size_t HUF_decompress1X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X2_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2 (DCtx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X2_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); +} + + +static size_t HUF_decompress4X2_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + /* Check */ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable + 1; + const HUF_DEltX2* const dt = (const HUF_DEltX2*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + const size_t segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) && (op4<(oend-7)) ; ) { + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_1(op1, &bitD1); + HUF_DECODE_SYMBOLX2_1(op2, &bitD2); + HUF_DECODE_SYMBOLX2_1(op3, &bitD3); + HUF_DECODE_SYMBOLX2_1(op4, &bitD4); + HUF_DECODE_SYMBOLX2_2(op1, &bitD1); + HUF_DECODE_SYMBOLX2_2(op2, &bitD2); + HUF_DECODE_SYMBOLX2_2(op3, &bitD3); + HUF_DECODE_SYMBOLX2_2(op4, &bitD4); + HUF_DECODE_SYMBOLX2_0(op1, &bitD1); + HUF_DECODE_SYMBOLX2_0(op2, &bitD2); + HUF_DECODE_SYMBOLX2_0(op3, &bitD3); + HUF_DECODE_SYMBOLX2_0(op4, &bitD4); + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 supposed already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX2(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX2(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX2(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX2(op4, &bitD4, oend, dt, dtLog); + + /* check */ + endSignal = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endSignal) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X2_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 0) return ERROR(GENERIC); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X2_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX2 (dctx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X2_usingDTable_internal (dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* *************************/ +/* double-symbols decoding */ +/* *************************/ +typedef struct { U16 sequence; BYTE nbBits; BYTE length; } HUF_DEltX4; /* double-symbols decoding */ + +typedef struct { BYTE symbol; BYTE weight; } sortedSymbol_t; + +/* HUF_fillDTableX4Level2() : + * `rankValOrigin` must be a table of at least (HUF_TABLELOG_MAX + 1) U32 */ +static void HUF_fillDTableX4Level2(HUF_DEltX4* DTable, U32 sizeLog, const U32 consumed, + const U32* rankValOrigin, const int minWeight, + const sortedSymbol_t* sortedSymbols, const U32 sortedListSize, + U32 nbBitsBaseline, U16 baseSeq) +{ + HUF_DEltX4 DElt; + U32 rankVal[HUF_TABLELOG_MAX + 1]; + + /* get pre-calculated rankVal */ + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill skipped values */ + if (minWeight>1) { + U32 i, skipSize = rankVal[minWeight]; + MEM_writeLE16(&(DElt.sequence), baseSeq); + DElt.nbBits = (BYTE)(consumed); + DElt.length = 1; + for (i = 0; i < skipSize; i++) + DTable[i] = DElt; + } + + /* fill DTable */ + { U32 s; for (s=0; s= 1 */ + + rankVal[weight] += length; + } } +} + +typedef U32 rankVal_t[HUF_TABLELOG_MAX][HUF_TABLELOG_MAX + 1]; + +static void HUF_fillDTableX4(HUF_DEltX4* DTable, const U32 targetLog, + const sortedSymbol_t* sortedList, const U32 sortedListSize, + const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32 nbBitsBaseline) +{ + U32 rankVal[HUF_TABLELOG_MAX + 1]; + const int scaleLog = nbBitsBaseline - targetLog; /* note : targetLog >= srcLog, hence scaleLog <= 1 */ + const U32 minBits = nbBitsBaseline - maxWeight; + U32 s; + + memcpy(rankVal, rankValOrigin, sizeof(rankVal)); + + /* fill DTable */ + for (s=0; s= minBits) { /* enough room for a second symbol */ + U32 sortedRank; + int minWeight = nbBits + scaleLog; + if (minWeight < 1) minWeight = 1; + sortedRank = rankStart[minWeight]; + HUF_fillDTableX4Level2(DTable+start, targetLog-nbBits, nbBits, + rankValOrigin[nbBits], minWeight, + sortedList+sortedRank, sortedListSize-sortedRank, + nbBitsBaseline, symbol); + } else { + HUF_DEltX4 DElt; + MEM_writeLE16(&(DElt.sequence), symbol); + DElt.nbBits = (BYTE)(nbBits); + DElt.length = 1; + { U32 const end = start + length; + U32 u; + for (u = start; u < end; u++) DTable[u] = DElt; + } } + rankVal[weight] += length; + } +} + +size_t HUF_readDTableX4 (HUF_DTable* DTable, const void* src, size_t srcSize) +{ + BYTE weightList[HUF_SYMBOLVALUE_MAX + 1]; + sortedSymbol_t sortedSymbol[HUF_SYMBOLVALUE_MAX + 1]; + U32 rankStats[HUF_TABLELOG_MAX + 1] = { 0 }; + U32 rankStart0[HUF_TABLELOG_MAX + 2] = { 0 }; + U32* const rankStart = rankStart0+1; + rankVal_t rankVal; + U32 tableLog, maxW, sizeOfSort, nbSymbols; + DTableDesc dtd = HUF_getDTableDesc(DTable); + U32 const maxTableLog = dtd.maxTableLog; + size_t iSize; + void* dtPtr = DTable+1; /* force compiler to avoid strict-aliasing */ + HUF_DEltX4* const dt = (HUF_DEltX4*)dtPtr; + + HUF_STATIC_ASSERT(sizeof(HUF_DEltX4) == sizeof(HUF_DTable)); /* if compilation fails here, assertion is false */ + if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); + /* memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ + + iSize = HUF_readStats(weightList, HUF_SYMBOLVALUE_MAX + 1, rankStats, &nbSymbols, &tableLog, src, srcSize); + if (HUF_isError(iSize)) return iSize; + + /* check result */ + if (tableLog > maxTableLog) return ERROR(tableLog_tooLarge); /* DTable can't fit code depth */ + + /* find maxWeight */ + for (maxW = tableLog; rankStats[maxW]==0; maxW--) {} /* necessarily finds a solution before 0 */ + + /* Get start index of each weight */ + { U32 w, nextRankStart = 0; + for (w=1; w> consumed; + } } } } + + HUF_fillDTableX4(dt, maxTableLog, + sortedSymbol, sizeOfSort, + rankStart0, rankVal, maxW, + tableLog+1); + + dtd.tableLog = (BYTE)maxTableLog; + dtd.tableType = 1; + memcpy(DTable, &dtd, sizeof(dtd)); + return iSize; +} + + +static U32 HUF_decodeSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 2); + BIT_skipBits(DStream, dt[val].nbBits); + return dt[val].length; +} + +static U32 HUF_decodeLastSymbolX4(void* op, BIT_DStream_t* DStream, const HUF_DEltX4* dt, const U32 dtLog) +{ + size_t const val = BIT_lookBitsFast(DStream, dtLog); /* note : dtLog >= 1 */ + memcpy(op, dt+val, 1); + if (dt[val].length==1) BIT_skipBits(DStream, dt[val].nbBits); + else { + if (DStream->bitsConsumed < (sizeof(DStream->bitContainer)*8)) { + BIT_skipBits(DStream, dt[val].nbBits); + if (DStream->bitsConsumed > (sizeof(DStream->bitContainer)*8)) + DStream->bitsConsumed = (sizeof(DStream->bitContainer)*8); /* ugly hack; works only because it's the last symbol. Note : can't easily extract nbBits from just this symbol */ + } } + return 1; +} + + +#define HUF_DECODE_SYMBOLX4_0(ptr, DStreamPtr) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_1(ptr, DStreamPtr) \ + if (MEM_64bits() || (HUF_TABLELOG_MAX<=12)) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +#define HUF_DECODE_SYMBOLX4_2(ptr, DStreamPtr) \ + if (MEM_64bits()) \ + ptr += HUF_decodeSymbolX4(ptr, DStreamPtr, dt, dtLog) + +static inline size_t HUF_decodeStreamX4(BYTE* p, BIT_DStream_t* bitDPtr, BYTE* const pEnd, const HUF_DEltX4* const dt, const U32 dtLog) +{ + BYTE* const pStart = p; + + /* up to 8 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p < pEnd-(sizeof(bitDPtr->bitContainer)-1))) { + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_1(p, bitDPtr); + HUF_DECODE_SYMBOLX4_2(p, bitDPtr); + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + } + + /* closer to end : up to 2 symbols at a time */ + while ((BIT_reloadDStream(bitDPtr) == BIT_DStream_unfinished) & (p <= pEnd-2)) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); + + while (p <= pEnd-2) + HUF_DECODE_SYMBOLX4_0(p, bitDPtr); /* no need to reload : reached the end of DStream */ + + if (p < pEnd) + p += HUF_decodeLastSymbolX4(p, bitDPtr, dt, dtLog); + + return p-pStart; +} + + +static size_t HUF_decompress1X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + BIT_DStream_t bitD; + + /* Init */ + { size_t const errorCode = BIT_initDStream(&bitD, cSrc, cSrcSize); + if (HUF_isError(errorCode)) return errorCode; + } + + /* decode */ + { BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; /* force compiler to not use strict-aliasing */ + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + HUF_decodeStreamX4(ostart, &bitD, oend, dt, dtd.tableLog); + } + + /* check */ + if (!BIT_endOfDStream(&bitD)) return ERROR(corruption_detected); + + /* decoded size */ + return dstSize; +} + +size_t HUF_decompress1X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress1X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress1X4_DCtx (HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t const hSize = HUF_readDTableX4 (DCtx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress1X4_usingDTable_internal (dst, dstSize, ip, cSrcSize, DCtx); +} + +size_t HUF_decompress1X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress1X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + +static size_t HUF_decompress4X4_usingDTable_internal( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + if (cSrcSize < 10) return ERROR(corruption_detected); /* strict minimum : jump table + 1 byte per stream */ + + { const BYTE* const istart = (const BYTE*) cSrc; + BYTE* const ostart = (BYTE*) dst; + BYTE* const oend = ostart + dstSize; + const void* const dtPtr = DTable+1; + const HUF_DEltX4* const dt = (const HUF_DEltX4*)dtPtr; + + /* Init */ + BIT_DStream_t bitD1; + BIT_DStream_t bitD2; + BIT_DStream_t bitD3; + BIT_DStream_t bitD4; + size_t const length1 = MEM_readLE16(istart); + size_t const length2 = MEM_readLE16(istart+2); + size_t const length3 = MEM_readLE16(istart+4); + size_t const length4 = cSrcSize - (length1 + length2 + length3 + 6); + const BYTE* const istart1 = istart + 6; /* jumpTable */ + const BYTE* const istart2 = istart1 + length1; + const BYTE* const istart3 = istart2 + length2; + const BYTE* const istart4 = istart3 + length3; + size_t const segmentSize = (dstSize+3) / 4; + BYTE* const opStart2 = ostart + segmentSize; + BYTE* const opStart3 = opStart2 + segmentSize; + BYTE* const opStart4 = opStart3 + segmentSize; + BYTE* op1 = ostart; + BYTE* op2 = opStart2; + BYTE* op3 = opStart3; + BYTE* op4 = opStart4; + U32 endSignal; + DTableDesc const dtd = HUF_getDTableDesc(DTable); + U32 const dtLog = dtd.tableLog; + + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + { size_t const errorCode = BIT_initDStream(&bitD1, istart1, length1); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD2, istart2, length2); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD3, istart3, length3); + if (HUF_isError(errorCode)) return errorCode; } + { size_t const errorCode = BIT_initDStream(&bitD4, istart4, length4); + if (HUF_isError(errorCode)) return errorCode; } + + /* 16-32 symbols per loop (4-8 symbols per stream) */ + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + for ( ; (endSignal==BIT_DStream_unfinished) & (op4<(oend-(sizeof(bitD4.bitContainer)-1))) ; ) { + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_1(op1, &bitD1); + HUF_DECODE_SYMBOLX4_1(op2, &bitD2); + HUF_DECODE_SYMBOLX4_1(op3, &bitD3); + HUF_DECODE_SYMBOLX4_1(op4, &bitD4); + HUF_DECODE_SYMBOLX4_2(op1, &bitD1); + HUF_DECODE_SYMBOLX4_2(op2, &bitD2); + HUF_DECODE_SYMBOLX4_2(op3, &bitD3); + HUF_DECODE_SYMBOLX4_2(op4, &bitD4); + HUF_DECODE_SYMBOLX4_0(op1, &bitD1); + HUF_DECODE_SYMBOLX4_0(op2, &bitD2); + HUF_DECODE_SYMBOLX4_0(op3, &bitD3); + HUF_DECODE_SYMBOLX4_0(op4, &bitD4); + + endSignal = BIT_reloadDStream(&bitD1) | BIT_reloadDStream(&bitD2) | BIT_reloadDStream(&bitD3) | BIT_reloadDStream(&bitD4); + } + + /* check corruption */ + if (op1 > opStart2) return ERROR(corruption_detected); + if (op2 > opStart3) return ERROR(corruption_detected); + if (op3 > opStart4) return ERROR(corruption_detected); + /* note : op4 already verified within main loop */ + + /* finish bitStreams one by one */ + HUF_decodeStreamX4(op1, &bitD1, opStart2, dt, dtLog); + HUF_decodeStreamX4(op2, &bitD2, opStart3, dt, dtLog); + HUF_decodeStreamX4(op3, &bitD3, opStart4, dt, dtLog); + HUF_decodeStreamX4(op4, &bitD4, oend, dt, dtLog); + + /* check */ + { U32 const endCheck = BIT_endOfDStream(&bitD1) & BIT_endOfDStream(&bitD2) & BIT_endOfDStream(&bitD3) & BIT_endOfDStream(&bitD4); + if (!endCheck) return ERROR(corruption_detected); } + + /* decoded size */ + return dstSize; + } +} + + +size_t HUF_decompress4X4_usingDTable( + void* dst, size_t dstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc dtd = HUF_getDTableDesc(DTable); + if (dtd.tableType != 1) return ERROR(GENERIC); + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable); +} + + +size_t HUF_decompress4X4_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + const BYTE* ip = (const BYTE*) cSrc; + + size_t hSize = HUF_readDTableX4 (dctx, cSrc, cSrcSize); + if (HUF_isError(hSize)) return hSize; + if (hSize >= cSrcSize) return ERROR(srcSize_wrong); + ip += hSize; cSrcSize -= hSize; + + return HUF_decompress4X4_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx); +} + +size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + HUF_CREATE_STATIC_DTABLEX4(DTable, HUF_TABLELOG_MAX); + return HUF_decompress4X4_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); +} + + +/* ********************************/ +/* Generic decompression selector */ +/* ********************************/ + +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress1X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, + const void* cSrc, size_t cSrcSize, + const HUF_DTable* DTable) +{ + DTableDesc const dtd = HUF_getDTableDesc(DTable); + return dtd.tableType ? HUF_decompress4X4_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable) : + HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable); +} + + +typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; +static const algo_time_t algoTime[16 /* Quantization */][3 /* single, double, quad */] = +{ + /* single, double, quad */ + {{0,0}, {1,1}, {2,2}}, /* Q==0 : impossible */ + {{0,0}, {1,1}, {2,2}}, /* Q==1 : impossible */ + {{ 38,130}, {1313, 74}, {2151, 38}}, /* Q == 2 : 12-18% */ + {{ 448,128}, {1353, 74}, {2238, 41}}, /* Q == 3 : 18-25% */ + {{ 556,128}, {1353, 74}, {2238, 47}}, /* Q == 4 : 25-32% */ + {{ 714,128}, {1418, 74}, {2436, 53}}, /* Q == 5 : 32-38% */ + {{ 883,128}, {1437, 74}, {2464, 61}}, /* Q == 6 : 38-44% */ + {{ 897,128}, {1515, 75}, {2622, 68}}, /* Q == 7 : 44-50% */ + {{ 926,128}, {1613, 75}, {2730, 75}}, /* Q == 8 : 50-56% */ + {{ 947,128}, {1729, 77}, {3359, 77}}, /* Q == 9 : 56-62% */ + {{1107,128}, {2083, 81}, {4006, 84}}, /* Q ==10 : 62-69% */ + {{1177,128}, {2379, 87}, {4785, 88}}, /* Q ==11 : 69-75% */ + {{1242,128}, {2415, 93}, {5155, 84}}, /* Q ==12 : 75-81% */ + {{1349,128}, {2644,106}, {5260,106}}, /* Q ==13 : 81-87% */ + {{1455,128}, {2422,124}, {4174,124}}, /* Q ==14 : 87-93% */ + {{ 722,128}, {1891,145}, {1936,146}}, /* Q ==15 : 93-99% */ +}; + +/** HUF_selectDecoder() : +* Tells which decoder is likely to decode faster, +* based on a set of pre-determined metrics. +* @return : 0==HUF_decompress4X2, 1==HUF_decompress4X4 . +* Assumption : 0 < cSrcSize < dstSize <= 128 KB */ +U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) +{ + /* decoder timing evaluation */ + U32 const Q = (U32)(cSrcSize * 16 / dstSize); /* Q < 16 since dstSize > cSrcSize */ + U32 const D256 = (U32)(dstSize >> 8); + U32 const DTime0 = algoTime[Q][0].tableTime + (algoTime[Q][0].decode256Time * D256); + U32 DTime1 = algoTime[Q][1].tableTime + (algoTime[Q][1].decode256Time * D256); + DTime1 += DTime1 >> 3; /* advantage to algorithm using less memory, for cache eviction */ + + return DTime1 < DTime0; +} + + +typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); + +size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + static const decompressionAlgo decompress[2] = { HUF_decompress4X2, HUF_decompress4X4 }; + + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); + } +} + +size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress4X_hufOnly (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if ((cSrcSize >= dstSize) || (cSrcSize <= 1)) return ERROR(corruption_detected); /* invalid */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress4X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} + +size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) +{ + /* validation checks */ + if (dstSize == 0) return ERROR(dstSize_tooSmall); + if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ + if (cSrcSize == dstSize) { memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ + if (cSrcSize == 1) { memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ + + { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); + return algoNb ? HUF_decompress1X4_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : + HUF_decompress1X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/decompress/zstd_decompress.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/decompress/zstd_decompress.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/decompress/zstd_decompress.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/decompress/zstd_decompress.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,2154 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/* *************************************************************** +* Tuning parameters +*****************************************************************/ +/*! + * HEAPMODE : + * Select how default decompression function ZSTD_decompress() will allocate memory, + * in memory stack (0), or in memory heap (1, requires malloc()) + */ +#ifndef ZSTD_HEAPMODE +# define ZSTD_HEAPMODE 1 +#endif + +/*! +* LEGACY_SUPPORT : +* if set to 1, ZSTD_decompress() can decode older formats (v0.1+) +*/ +#ifndef ZSTD_LEGACY_SUPPORT +# define ZSTD_LEGACY_SUPPORT 0 +#endif + +/*! +* MAXWINDOWSIZE_DEFAULT : +* maximum window size accepted by DStream, by default. +* Frames requiring more memory will be rejected. +*/ +#ifndef ZSTD_MAXWINDOWSIZE_DEFAULT +# define ZSTD_MAXWINDOWSIZE_DEFAULT ((1 << ZSTD_WINDOWLOG_MAX) + 1) /* defined within zstd.h */ +#endif + + +/*-******************************************************* +* Dependencies +*********************************************************/ +#include /* memcpy, memmove, memset */ +#include "mem.h" /* low level memory routines */ +#define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ +#include "xxhash.h" /* XXH64_* */ +#define FSE_STATIC_LINKING_ONLY +#include "fse.h" +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) +# include "zstd_legacy.h" +#endif + + +#if defined(_MSC_VER) +# include /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */ +# define ZSTD_PREFETCH(ptr) _mm_prefetch((const char*)ptr, _MM_HINT_T0) +#elif defined(__GNUC__) +# define ZSTD_PREFETCH(ptr) __builtin_prefetch(ptr, 0, 0) +#else +# define ZSTD_PREFETCH(ptr) /* disabled */ +#endif + +/*-************************************* +* Macros +***************************************/ +#define ZSTD_isError ERR_isError /* for inlining */ +#define FSE_isError ERR_isError +#define HUF_isError ERR_isError + + +/*_******************************************************* +* Memory operations +**********************************************************/ +static void ZSTD_copy4(void* dst, const void* src) { memcpy(dst, src, 4); } + + +/*-************************************************************* +* Context management +***************************************************************/ +typedef enum { ZSTDds_getFrameHeaderSize, ZSTDds_decodeFrameHeader, + ZSTDds_decodeBlockHeader, ZSTDds_decompressBlock, + ZSTDds_decompressLastBlock, ZSTDds_checkChecksum, + ZSTDds_decodeSkippableHeader, ZSTDds_skipFrame } ZSTD_dStage; + +struct ZSTD_DCtx_s +{ + const FSE_DTable* LLTptr; + const FSE_DTable* MLTptr; + const FSE_DTable* OFTptr; + const HUF_DTable* HUFptr; + FSE_DTable LLTable[FSE_DTABLE_SIZE_U32(LLFSELog)]; + FSE_DTable OFTable[FSE_DTABLE_SIZE_U32(OffFSELog)]; + FSE_DTable MLTable[FSE_DTABLE_SIZE_U32(MLFSELog)]; + HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + const void* previousDstEnd; + const void* base; + const void* vBase; + const void* dictEnd; + size_t expected; + U32 rep[ZSTD_REP_NUM]; + ZSTD_frameParams fParams; + blockType_e bType; /* used in ZSTD_decompressContinue(), to transfer blockType between header decoding and block decoding stages */ + ZSTD_dStage stage; + U32 litEntropy; + U32 fseEntropy; + XXH64_state_t xxhState; + size_t headerSize; + U32 dictID; + const BYTE* litPtr; + ZSTD_customMem customMem; + size_t litSize; + size_t rleSize; + BYTE litBuffer[ZSTD_BLOCKSIZE_ABSOLUTEMAX + WILDCOPY_OVERLENGTH]; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; +}; /* typedef'd to ZSTD_DCtx within "zstd.h" */ + +size_t ZSTD_sizeof_DCtx (const ZSTD_DCtx* dctx) { return (dctx==NULL) ? 0 : sizeof(ZSTD_DCtx); } + +size_t ZSTD_estimateDCtxSize(void) { return sizeof(ZSTD_DCtx); } + +size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) +{ + dctx->expected = ZSTD_frameHeaderSize_prefix; + dctx->stage = ZSTDds_getFrameHeaderSize; + dctx->previousDstEnd = NULL; + dctx->base = NULL; + dctx->vBase = NULL; + dctx->dictEnd = NULL; + dctx->hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->litEntropy = dctx->fseEntropy = 0; + dctx->dictID = 0; + MEM_STATIC_ASSERT(sizeof(dctx->rep) == sizeof(repStartValue)); + memcpy(dctx->rep, repStartValue, sizeof(repStartValue)); /* initial repcodes */ + dctx->LLTptr = dctx->LLTable; + dctx->MLTptr = dctx->MLTable; + dctx->OFTptr = dctx->OFTable; + dctx->HUFptr = dctx->hufTable; + return 0; +} + +ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem) +{ + ZSTD_DCtx* dctx; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + dctx = (ZSTD_DCtx*)ZSTD_malloc(sizeof(ZSTD_DCtx), customMem); + if (!dctx) return NULL; + memcpy(&dctx->customMem, &customMem, sizeof(customMem)); + ZSTD_decompressBegin(dctx); + return dctx; +} + +ZSTD_DCtx* ZSTD_createDCtx(void) +{ + return ZSTD_createDCtx_advanced(defaultCustomMem); +} + +size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx) +{ + if (dctx==NULL) return 0; /* support free on NULL */ + ZSTD_free(dctx, dctx->customMem); + return 0; /* reserved as a potential error code in the future */ +} + +void ZSTD_copyDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + size_t const workSpaceSize = (ZSTD_BLOCKSIZE_ABSOLUTEMAX+WILDCOPY_OVERLENGTH) + ZSTD_frameHeaderSize_max; + memcpy(dstDCtx, srcDCtx, sizeof(ZSTD_DCtx) - workSpaceSize); /* no need to copy workspace */ +} + +static void ZSTD_refDCtx(ZSTD_DCtx* dstDCtx, const ZSTD_DCtx* srcDCtx) +{ + ZSTD_decompressBegin(dstDCtx); /* init */ + if (srcDCtx) { /* support refDCtx on NULL */ + dstDCtx->dictEnd = srcDCtx->dictEnd; + dstDCtx->vBase = srcDCtx->vBase; + dstDCtx->base = srcDCtx->base; + dstDCtx->previousDstEnd = srcDCtx->previousDstEnd; + dstDCtx->dictID = srcDCtx->dictID; + dstDCtx->litEntropy = srcDCtx->litEntropy; + dstDCtx->fseEntropy = srcDCtx->fseEntropy; + dstDCtx->LLTptr = srcDCtx->LLTable; + dstDCtx->MLTptr = srcDCtx->MLTable; + dstDCtx->OFTptr = srcDCtx->OFTable; + dstDCtx->HUFptr = srcDCtx->hufTable; + dstDCtx->rep[0] = srcDCtx->rep[0]; + dstDCtx->rep[1] = srcDCtx->rep[1]; + dstDCtx->rep[2] = srcDCtx->rep[2]; + } +} + + +/*-************************************************************* +* Decompression section +***************************************************************/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +unsigned ZSTD_isFrame(const void* buffer, size_t size) +{ + if (size < 4) return 0; + { U32 const magic = MEM_readLE32(buffer); + if (magic == ZSTD_MAGICNUMBER) return 1; + if ((magic & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) return 1; + } +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (ZSTD_isLegacy(buffer, size)) return 1; +#endif + return 0; +} + + +/** ZSTD_frameHeaderSize() : +* srcSize must be >= ZSTD_frameHeaderSize_prefix. +* @return : size of the Frame Header */ +static size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) +{ + if (srcSize < ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); + { BYTE const fhd = ((const BYTE*)src)[4]; + U32 const dictID= fhd & 3; + U32 const singleSegment = (fhd >> 5) & 1; + U32 const fcsId = fhd >> 6; + return ZSTD_frameHeaderSize_prefix + !singleSegment + ZSTD_did_fieldSize[dictID] + ZSTD_fcs_fieldSize[fcsId] + + (singleSegment && !fcsId); + } +} + + +/** ZSTD_getFrameParams() : +* decode Frame Header, or require larger `srcSize`. +* @return : 0, `fparamsPtr` is correctly filled, +* >0, `srcSize` is too small, result is expected `srcSize`, +* or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + + if (srcSize < ZSTD_frameHeaderSize_prefix) return ZSTD_frameHeaderSize_prefix; + if (MEM_readLE32(src) != ZSTD_MAGICNUMBER) { + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { + if (srcSize < ZSTD_skippableHeaderSize) return ZSTD_skippableHeaderSize; /* magic number + skippable frame length */ + memset(fparamsPtr, 0, sizeof(*fparamsPtr)); + fparamsPtr->frameContentSize = MEM_readLE32((const char *)src + 4); + fparamsPtr->windowSize = 0; /* windowSize==0 means a frame is skippable */ + return 0; + } + return ERROR(prefix_unknown); + } + + /* ensure there is enough `srcSize` to fully read/decode frame header */ + { size_t const fhsize = ZSTD_frameHeaderSize(src, srcSize); + if (srcSize < fhsize) return fhsize; } + + { BYTE const fhdByte = ip[4]; + size_t pos = 5; + U32 const dictIDSizeCode = fhdByte&3; + U32 const checksumFlag = (fhdByte>>2)&1; + U32 const singleSegment = (fhdByte>>5)&1; + U32 const fcsID = fhdByte>>6; + U32 const windowSizeMax = 1U << ZSTD_WINDOWLOG_MAX; + U32 windowSize = 0; + U32 dictID = 0; + U64 frameContentSize = 0; + if ((fhdByte & 0x08) != 0) return ERROR(frameParameter_unsupported); /* reserved bits, which must be zero */ + if (!singleSegment) { + BYTE const wlByte = ip[pos++]; + U32 const windowLog = (wlByte >> 3) + ZSTD_WINDOWLOG_ABSOLUTEMIN; + if (windowLog > ZSTD_WINDOWLOG_MAX) return ERROR(frameParameter_windowTooLarge); /* avoids issue with 1 << windowLog */ + windowSize = (1U << windowLog); + windowSize += (windowSize >> 3) * (wlByte&7); + } + + switch(dictIDSizeCode) + { + default: /* impossible */ + case 0 : break; + case 1 : dictID = ip[pos]; pos++; break; + case 2 : dictID = MEM_readLE16(ip+pos); pos+=2; break; + case 3 : dictID = MEM_readLE32(ip+pos); pos+=4; break; + } + switch(fcsID) + { + default: /* impossible */ + case 0 : if (singleSegment) frameContentSize = ip[pos]; break; + case 1 : frameContentSize = MEM_readLE16(ip+pos)+256; break; + case 2 : frameContentSize = MEM_readLE32(ip+pos); break; + case 3 : frameContentSize = MEM_readLE64(ip+pos); break; + } + if (!windowSize) windowSize = (U32)frameContentSize; + if (windowSize > windowSizeMax) return ERROR(frameParameter_windowTooLarge); + fparamsPtr->frameContentSize = frameContentSize; + fparamsPtr->windowSize = windowSize; + fparamsPtr->dictID = dictID; + fparamsPtr->checksumFlag = checksumFlag; + } + return 0; +} + + +/** ZSTD_getDecompressedSize() : +* compatible with legacy mode +* @return : decompressed size if known, 0 otherwise + note : 0 can mean any of the following : + - decompressed size is not present within frame header + - frame header unknown / not supported + - frame header not complete (`srcSize` too small) */ +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_getDecompressedSize_legacy(src, srcSize); +#endif + { ZSTD_frameParams fparams; + size_t const frResult = ZSTD_getFrameParams(&fparams, src, srcSize); + if (frResult!=0) return 0; + return fparams.frameContentSize; + } +} + + +/** ZSTD_decodeFrameHeader() : +* `headerSize` must be the size provided by ZSTD_frameHeaderSize(). +* @return : 0 if success, or an error code, which can be tested using ZSTD_isError() */ +static size_t ZSTD_decodeFrameHeader(ZSTD_DCtx* dctx, const void* src, size_t headerSize) +{ + size_t const result = ZSTD_getFrameParams(&(dctx->fParams), src, headerSize); + if (ZSTD_isError(result)) return result; /* invalid header */ + if (result>0) return ERROR(srcSize_wrong); /* headerSize too small */ + if (dctx->fParams.dictID && (dctx->dictID != dctx->fParams.dictID)) return ERROR(dictionary_wrong); + if (dctx->fParams.checksumFlag) XXH64_reset(&dctx->xxhState, 0); + return 0; +} + + +typedef struct +{ + blockType_e blockType; + U32 lastBlock; + U32 origSize; +} blockProperties_t; + +/*! ZSTD_getcBlockSize() : +* Provides the size of compressed block from block header `src` */ +size_t ZSTD_getcBlockSize(const void* src, size_t srcSize, blockProperties_t* bpPtr) +{ + if (srcSize < ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + { U32 const cBlockHeader = MEM_readLE24(src); + U32 const cSize = cBlockHeader >> 3; + bpPtr->lastBlock = cBlockHeader & 1; + bpPtr->blockType = (blockType_e)((cBlockHeader >> 1) & 3); + bpPtr->origSize = cSize; /* only useful for RLE */ + if (bpPtr->blockType == bt_rle) return 1; + if (bpPtr->blockType == bt_reserved) return ERROR(corruption_detected); + return cSize; + } +} + + +static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + if (srcSize > dstCapacity) return ERROR(dstSize_tooSmall); + memcpy(dst, src, srcSize); + return srcSize; +} + + +static size_t ZSTD_setRleBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, size_t regenSize) +{ + if (srcSize != 1) return ERROR(srcSize_wrong); + if (regenSize > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, *(const BYTE*)src, regenSize); + return regenSize; +} + +/*! ZSTD_decodeLiteralsBlock() : + @return : nb of bytes read from src (< srcSize ) */ +size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, + const void* src, size_t srcSize) /* note : srcSize < BLOCKSIZE */ +{ + if (srcSize < MIN_CBLOCK_SIZE) return ERROR(corruption_detected); + + { const BYTE* const istart = (const BYTE*) src; + symbolEncodingType_e const litEncType = (symbolEncodingType_e)(istart[0] & 3); + + switch(litEncType) + { + case set_repeat: + if (dctx->litEntropy==0) return ERROR(dictionary_corrupted); + /* fall-through */ + case set_compressed: + if (srcSize < 5) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3 */ + { size_t lhSize, litSize, litCSize; + U32 singleStream=0; + U32 const lhlCode = (istart[0] >> 2) & 3; + U32 const lhc = MEM_readLE32(istart); + switch(lhlCode) + { + case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ + /* 2 - 2 - 10 - 10 */ + singleStream = !lhlCode; + lhSize = 3; + litSize = (lhc >> 4) & 0x3FF; + litCSize = (lhc >> 14) & 0x3FF; + break; + case 2: + /* 2 - 2 - 14 - 14 */ + lhSize = 4; + litSize = (lhc >> 4) & 0x3FFF; + litCSize = lhc >> 18; + break; + case 3: + /* 2 - 2 - 18 - 18 */ + lhSize = 5; + litSize = (lhc >> 4) & 0x3FFFF; + litCSize = (lhc >> 22) + (istart[4] << 10); + break; + } + if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + if (litCSize + lhSize > srcSize) return ERROR(corruption_detected); + + if (HUF_isError((litEncType==set_repeat) ? + ( singleStream ? + HUF_decompress1X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) : + HUF_decompress4X_usingDTable(dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->HUFptr) ) : + ( singleStream ? + HUF_decompress1X2_DCtx(dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize) : + HUF_decompress4X_hufOnly (dctx->hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize)) )) + return ERROR(corruption_detected); + + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + dctx->litEntropy = 1; + if (litEncType==set_compressed) dctx->HUFptr = dctx->hufTable; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return litCSize + lhSize; + } + + case set_basic: + { size_t litSize, lhSize; + U32 const lhlCode = ((istart[0]) >> 2) & 3; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + break; + } + + if (lhSize+litSize+WILDCOPY_OVERLENGTH > srcSize) { /* risk reading beyond src buffer with wildcopy */ + if (litSize+lhSize > srcSize) return ERROR(corruption_detected); + memcpy(dctx->litBuffer, istart+lhSize, litSize); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + memset(dctx->litBuffer + dctx->litSize, 0, WILDCOPY_OVERLENGTH); + return lhSize+litSize; + } + /* direct reference into compressed stream */ + dctx->litPtr = istart+lhSize; + dctx->litSize = litSize; + return lhSize+litSize; + } + + case set_rle: + { U32 const lhlCode = ((istart[0]) >> 2) & 3; + size_t litSize, lhSize; + switch(lhlCode) + { + case 0: case 2: default: /* note : default is impossible, since lhlCode into [0..3] */ + lhSize = 1; + litSize = istart[0] >> 3; + break; + case 1: + lhSize = 2; + litSize = MEM_readLE16(istart) >> 4; + break; + case 3: + lhSize = 3; + litSize = MEM_readLE24(istart) >> 4; + if (srcSize<4) return ERROR(corruption_detected); /* srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4 */ + break; + } + if (litSize > ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(corruption_detected); + memset(dctx->litBuffer, istart[lhSize], litSize + WILDCOPY_OVERLENGTH); + dctx->litPtr = dctx->litBuffer; + dctx->litSize = litSize; + return lhSize+1; + } + default: + return ERROR(corruption_detected); /* impossible */ + } + } +} + + +typedef union { + FSE_decode_t realData; + U32 alignedBy4; +} FSE_decode_t4; + +static const FSE_decode_t4 LL_defaultDTable[(1< max) return ERROR(corruption_detected); + FSE_buildDTable_rle(DTableSpace, *(const BYTE*)src); + *DTablePtr = DTableSpace; + return 1; + case set_basic : + *DTablePtr = (const FSE_DTable*)tmpPtr; + return 0; + case set_repeat: + if (!flagRepeatTable) return ERROR(corruption_detected); + return 0; + default : /* impossible */ + case set_compressed : + { U32 tableLog; + S16 norm[MaxSeq+1]; + size_t const headerSize = FSE_readNCount(norm, &max, &tableLog, src, srcSize); + if (FSE_isError(headerSize)) return ERROR(corruption_detected); + if (tableLog > maxLog) return ERROR(corruption_detected); + FSE_buildDTable(DTableSpace, norm, max, tableLog); + *DTablePtr = DTableSpace; + return headerSize; + } } +} + +size_t ZSTD_decodeSeqHeaders(ZSTD_DCtx* dctx, int* nbSeqPtr, + const void* src, size_t srcSize) +{ + const BYTE* const istart = (const BYTE* const)src; + const BYTE* const iend = istart + srcSize; + const BYTE* ip = istart; + + /* check */ + if (srcSize < MIN_SEQUENCES_SIZE) return ERROR(srcSize_wrong); + + /* SeqHead */ + { int nbSeq = *ip++; + if (!nbSeq) { *nbSeqPtr=0; return 1; } + if (nbSeq > 0x7F) { + if (nbSeq == 0xFF) { + if (ip+2 > iend) return ERROR(srcSize_wrong); + nbSeq = MEM_readLE16(ip) + LONGNBSEQ, ip+=2; + } else { + if (ip >= iend) return ERROR(srcSize_wrong); + nbSeq = ((nbSeq-0x80)<<8) + *ip++; + } + } + *nbSeqPtr = nbSeq; + } + + /* FSE table descriptors */ + if (ip+4 > iend) return ERROR(srcSize_wrong); /* minimum possible size */ + { symbolEncodingType_e const LLtype = (symbolEncodingType_e)(*ip >> 6); + symbolEncodingType_e const OFtype = (symbolEncodingType_e)((*ip >> 4) & 3); + symbolEncodingType_e const MLtype = (symbolEncodingType_e)((*ip >> 2) & 3); + ip++; + + /* Build DTables */ + { size_t const llhSize = ZSTD_buildSeqTable(dctx->LLTable, &dctx->LLTptr, + LLtype, MaxLL, LLFSELog, + ip, iend-ip, LL_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(llhSize)) return ERROR(corruption_detected); + ip += llhSize; + } + { size_t const ofhSize = ZSTD_buildSeqTable(dctx->OFTable, &dctx->OFTptr, + OFtype, MaxOff, OffFSELog, + ip, iend-ip, OF_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(ofhSize)) return ERROR(corruption_detected); + ip += ofhSize; + } + { size_t const mlhSize = ZSTD_buildSeqTable(dctx->MLTable, &dctx->MLTptr, + MLtype, MaxML, MLFSELog, + ip, iend-ip, ML_defaultDTable, dctx->fseEntropy); + if (ZSTD_isError(mlhSize)) return ERROR(corruption_detected); + ip += mlhSize; + } + } + + return ip-istart; +} + + +typedef struct { + size_t litLength; + size_t matchLength; + size_t offset; + const BYTE* match; +} seq_t; + +typedef struct { + BIT_DStream_t DStream; + FSE_DState_t stateLL; + FSE_DState_t stateOffb; + FSE_DState_t stateML; + size_t prevOffset[ZSTD_REP_NUM]; + const BYTE* base; + size_t pos; + iPtrDiff gotoDict; +} seqState_t; + + +FORCE_NOINLINE +size_t ZSTD_execSequenceLast7(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd <= oend_w) return ERROR(GENERIC); /* Precondition */ + + /* copy literals */ + if (op < oend_w) { + ZSTD_wildcopy(op, *litPtr, oend_w - op); + *litPtr += oend_w - op; + op = oend_w; + } + while (op < oLitEnd) *op++ = *(*litPtr)++; + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match = dictEnd - (base-match); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + } } + while (op < oMatchEnd) *op++ = *match++; + return sequenceLength; +} + + + + +static seq_t ZSTD_decodeSequence(seqState_t* seqState) +{ + seq_t seq; + + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + + if (ofCode <= 1) { + offset += (llCode==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); + + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits() || + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); + + /* ANS state update */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + + +FORCE_INLINE +size_t ZSTD_execSequence(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = oLitEnd - sequence.offset; + + /* check */ + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + match += (dictEnd-base); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + + +static size_t ZSTD_decompressSequences( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { + seqState_t seqState; + dctx->fseEntropy = 1; + { U32 i; for (i=0; irep[i]; } + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + for ( ; (BIT_reloadDStream(&(seqState.DStream)) <= BIT_DStream_completed) && nbSeq ; ) { + nbSeq--; + { seq_t const sequence = ZSTD_decodeSequence(&seqState); + size_t const oneSeqSize = ZSTD_execSequence(op, oend, sequence, &litPtr, litEnd, base, vBase, dictEnd); + if (ZSTD_isError(oneSeqSize)) return oneSeqSize; + op += oneSeqSize; + } } + + /* check if reached exact end */ + if (nbSeq) return ERROR(corruption_detected); + /* save reps for next block */ + { U32 i; for (i=0; irep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +static seq_t ZSTD_decodeSequenceLong(seqState_t* seqState) +{ + seq_t seq; + + U32 const llCode = FSE_peekSymbol(&seqState->stateLL); + U32 const mlCode = FSE_peekSymbol(&seqState->stateML); + U32 const ofCode = FSE_peekSymbol(&seqState->stateOffb); /* <= maxOff, by table construction */ + + U32 const llBits = LL_bits[llCode]; + U32 const mlBits = ML_bits[mlCode]; + U32 const ofBits = ofCode; + U32 const totalBits = llBits+mlBits+ofBits; + + static const U32 LL_base[MaxLL+1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 18, 20, 22, 24, 28, 32, 40, 48, 64, 0x80, 0x100, 0x200, 0x400, 0x800, 0x1000, + 0x2000, 0x4000, 0x8000, 0x10000 }; + + static const U32 ML_base[MaxML+1] = { + 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, + 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, + 35, 37, 39, 41, 43, 47, 51, 59, 67, 83, 99, 0x83, 0x103, 0x203, 0x403, 0x803, + 0x1003, 0x2003, 0x4003, 0x8003, 0x10003 }; + + static const U32 OF_base[MaxOff+1] = { + 0, 1, 1, 5, 0xD, 0x1D, 0x3D, 0x7D, + 0xFD, 0x1FD, 0x3FD, 0x7FD, 0xFFD, 0x1FFD, 0x3FFD, 0x7FFD, + 0xFFFD, 0x1FFFD, 0x3FFFD, 0x7FFFD, 0xFFFFD, 0x1FFFFD, 0x3FFFFD, 0x7FFFFD, + 0xFFFFFD, 0x1FFFFFD, 0x3FFFFFD, 0x7FFFFFD, 0xFFFFFFD }; + + /* sequence */ + { size_t offset; + if (!ofCode) + offset = 0; + else { + offset = OF_base[ofCode] + BIT_readBitsFast(&seqState->DStream, ofBits); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); + } + + if (ofCode <= 1) { + offset += (llCode==0); + if (offset) { + size_t temp = (offset==3) ? seqState->prevOffset[0] - 1 : seqState->prevOffset[offset]; + temp += !temp; /* 0 is not valid; input is corrupted; force offset to 1 */ + if (offset != 1) seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset = temp; + } else { + offset = seqState->prevOffset[0]; + } + } else { + seqState->prevOffset[2] = seqState->prevOffset[1]; + seqState->prevOffset[1] = seqState->prevOffset[0]; + seqState->prevOffset[0] = offset; + } + seq.offset = offset; + } + + seq.matchLength = ML_base[mlCode] + ((mlCode>31) ? BIT_readBitsFast(&seqState->DStream, mlBits) : 0); /* <= 16 bits */ + if (MEM_32bits() && (mlBits+llBits>24)) BIT_reloadDStream(&seqState->DStream); + + seq.litLength = LL_base[llCode] + ((llCode>15) ? BIT_readBitsFast(&seqState->DStream, llBits) : 0); /* <= 16 bits */ + if (MEM_32bits() || + (totalBits > 64 - 7 - (LLFSELog+MLFSELog+OffFSELog)) ) BIT_reloadDStream(&seqState->DStream); + + { size_t const pos = seqState->pos + seq.litLength; + seq.match = seqState->base + pos - seq.offset; /* single memory segment */ + if (seq.offset > pos) seq.match += seqState->gotoDict; /* separate memory segment */ + seqState->pos = pos + seq.matchLength; + } + + /* ANS state update */ + FSE_updateState(&seqState->stateLL, &seqState->DStream); /* <= 9 bits */ + FSE_updateState(&seqState->stateML, &seqState->DStream); /* <= 9 bits */ + if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); /* <= 18 bits */ + FSE_updateState(&seqState->stateOffb, &seqState->DStream); /* <= 8 bits */ + + return seq; +} + +FORCE_INLINE +size_t ZSTD_execSequenceLong(BYTE* op, + BYTE* const oend, seq_t sequence, + const BYTE** litPtr, const BYTE* const litLimit, + const BYTE* const base, const BYTE* const vBase, const BYTE* const dictEnd) +{ + BYTE* const oLitEnd = op + sequence.litLength; + size_t const sequenceLength = sequence.litLength + sequence.matchLength; + BYTE* const oMatchEnd = op + sequenceLength; /* risk : address space overflow (32-bits) */ + BYTE* const oend_w = oend - WILDCOPY_OVERLENGTH; + const BYTE* const iLitEnd = *litPtr + sequence.litLength; + const BYTE* match = sequence.match; + + /* check */ +#if 1 + if (oMatchEnd>oend) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (oLitEnd>oend_w) return ZSTD_execSequenceLast7(op, oend, sequence, litPtr, litLimit, base, vBase, dictEnd); +#endif + + /* copy Literals */ + ZSTD_copy8(op, *litPtr); + if (sequence.litLength > 8) + ZSTD_wildcopy(op+8, (*litPtr)+8, sequence.litLength - 8); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + op = oLitEnd; + *litPtr = iLitEnd; /* update for next sequence */ + + /* copy Match */ +#if 1 + if (sequence.offset > (size_t)(oLitEnd - base)) { + /* offset beyond prefix */ + if (sequence.offset > (size_t)(oLitEnd - vBase)) return ERROR(corruption_detected); + if (match + sequence.matchLength <= dictEnd) { + memmove(oLitEnd, match, sequence.matchLength); + return sequenceLength; + } + /* span extDict & currentPrefixSegment */ + { size_t const length1 = dictEnd - match; + memmove(oLitEnd, match, length1); + op = oLitEnd + length1; + sequence.matchLength -= length1; + match = base; + if (op > oend_w || sequence.matchLength < MINMATCH) { + U32 i; + for (i = 0; i < sequence.matchLength; ++i) op[i] = match[i]; + return sequenceLength; + } + } } + /* Requirement: op <= oend_w && sequence.matchLength >= MINMATCH */ +#endif + + /* match within prefix */ + if (sequence.offset < 8) { + /* close range match, overlap */ + static const U32 dec32table[] = { 0, 1, 2, 1, 4, 4, 4, 4 }; /* added */ + static const int dec64table[] = { 8, 8, 8, 7, 8, 9,10,11 }; /* substracted */ + int const sub2 = dec64table[sequence.offset]; + op[0] = match[0]; + op[1] = match[1]; + op[2] = match[2]; + op[3] = match[3]; + match += dec32table[sequence.offset]; + ZSTD_copy4(op+4, match); + match -= sub2; + } else { + ZSTD_copy8(op, match); + } + op += 8; match += 8; + + if (oMatchEnd > oend-(16-MINMATCH)) { + if (op < oend_w) { + ZSTD_wildcopy(op, match, oend_w - op); + match += oend_w - op; + op = oend_w; + } + while (op < oMatchEnd) *op++ = *match++; + } else { + ZSTD_wildcopy(op, match, (ptrdiff_t)sequence.matchLength-8); /* works even if matchLength < 8 */ + } + return sequenceLength; +} + +static size_t ZSTD_decompressSequencesLong( + ZSTD_DCtx* dctx, + void* dst, size_t maxDstSize, + const void* seqStart, size_t seqSize) +{ + const BYTE* ip = (const BYTE*)seqStart; + const BYTE* const iend = ip + seqSize; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + maxDstSize; + BYTE* op = ostart; + const BYTE* litPtr = dctx->litPtr; + const BYTE* const litEnd = litPtr + dctx->litSize; + const BYTE* const base = (const BYTE*) (dctx->base); + const BYTE* const vBase = (const BYTE*) (dctx->vBase); + const BYTE* const dictEnd = (const BYTE*) (dctx->dictEnd); + int nbSeq; + + /* Build Decoding Tables */ + { size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, seqSize); + if (ZSTD_isError(seqHSize)) return seqHSize; + ip += seqHSize; + } + + /* Regen sequences */ + if (nbSeq) { +#define STORED_SEQS 4 +#define STOSEQ_MASK (STORED_SEQS-1) +#define ADVANCED_SEQS 4 + seq_t sequences[STORED_SEQS]; + int const seqAdvance = MIN(nbSeq, ADVANCED_SEQS); + seqState_t seqState; + int seqNb; + dctx->fseEntropy = 1; + { U32 i; for (i=0; irep[i]; } + seqState.base = base; + seqState.pos = (size_t)(op-base); + seqState.gotoDict = (iPtrDiff)(dictEnd - base); + CHECK_E(BIT_initDStream(&seqState.DStream, ip, iend-ip), corruption_detected); + FSE_initDState(&seqState.stateLL, &seqState.DStream, dctx->LLTptr); + FSE_initDState(&seqState.stateOffb, &seqState.DStream, dctx->OFTptr); + FSE_initDState(&seqState.stateML, &seqState.DStream, dctx->MLTptr); + + /* prepare in advance */ + for (seqNb=0; (BIT_reloadDStream(&seqState.DStream) <= BIT_DStream_completed) && seqNbrep[i] = (U32)(seqState.prevOffset[i]); } + } + + /* last literal segment */ + { size_t const lastLLSize = litEnd - litPtr; + if (lastLLSize > (size_t)(oend-op)) return ERROR(dstSize_tooSmall); + memcpy(op, litPtr, lastLLSize); + op += lastLLSize; + } + + return op-ostart; +} + + +static size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ /* blockType == blockCompressed */ + const BYTE* ip = (const BYTE*)src; + + if (srcSize >= ZSTD_BLOCKSIZE_ABSOLUTEMAX) return ERROR(srcSize_wrong); + + /* Decode literals sub-block */ + { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize); + if (ZSTD_isError(litCSize)) return litCSize; + ip += litCSize; + srcSize -= litCSize; + } + if (dctx->fParams.windowSize > (1<<23)) return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize); + return ZSTD_decompressSequences(dctx, dst, dstCapacity, ip, srcSize); +} + + +static void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst) +{ + if (dst != dctx->previousDstEnd) { /* not contiguous */ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dst - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dst; + dctx->previousDstEnd = dst; + } +} + +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + size_t dSize; + ZSTD_checkContinuity(dctx, dst); + dSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + dctx->previousDstEnd = (char*)dst + dSize; + return dSize; +} + + +/** ZSTD_insertBlock() : + insert `src` block into `dctx` history. Useful to track uncompressed blocks. */ +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize) +{ + ZSTD_checkContinuity(dctx, blockStart); + dctx->previousDstEnd = (const char*)blockStart + blockSize; + return blockSize; +} + + +size_t ZSTD_generateNxBytes(void* dst, size_t dstCapacity, BYTE byte, size_t length) +{ + if (length > dstCapacity) return ERROR(dstSize_tooSmall); + memset(dst, byte, length); + return length; +} + + +/*! ZSTD_decompressFrame() : +* `dctx` must be properly initialized */ +static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + const BYTE* ip = (const BYTE*)src; + BYTE* const ostart = (BYTE* const)dst; + BYTE* const oend = ostart + dstCapacity; + BYTE* op = ostart; + size_t remainingSize = srcSize; + + /* check */ + if (srcSize < ZSTD_frameHeaderSize_min+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + + /* Frame Header */ + { size_t const frameHeaderSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); + if (ZSTD_isError(frameHeaderSize)) return frameHeaderSize; + if (srcSize < frameHeaderSize+ZSTD_blockHeaderSize) return ERROR(srcSize_wrong); + CHECK_F(ZSTD_decodeFrameHeader(dctx, src, frameHeaderSize)); + ip += frameHeaderSize; remainingSize -= frameHeaderSize; + } + + /* Loop on each block */ + while (1) { + size_t decodedSize; + blockProperties_t blockProperties; + size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSize, &blockProperties); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + + ip += ZSTD_blockHeaderSize; + remainingSize -= ZSTD_blockHeaderSize; + if (cBlockSize > remainingSize) return ERROR(srcSize_wrong); + + switch(blockProperties.blockType) + { + case bt_compressed: + decodedSize = ZSTD_decompressBlock_internal(dctx, op, oend-op, ip, cBlockSize); + break; + case bt_raw : + decodedSize = ZSTD_copyRawBlock(op, oend-op, ip, cBlockSize); + break; + case bt_rle : + decodedSize = ZSTD_generateNxBytes(op, oend-op, *ip, blockProperties.origSize); + break; + case bt_reserved : + default: + return ERROR(corruption_detected); + } + + if (ZSTD_isError(decodedSize)) return decodedSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, op, decodedSize); + op += decodedSize; + ip += cBlockSize; + remainingSize -= cBlockSize; + if (blockProperties.lastBlock) break; + } + + if (dctx->fParams.checksumFlag) { /* Frame content checksum verification */ + U32 const checkCalc = (U32)XXH64_digest(&dctx->xxhState); + U32 checkRead; + if (remainingSize<4) return ERROR(checksum_wrong); + checkRead = MEM_readLE32(ip); + if (checkRead != checkCalc) return ERROR(checksum_wrong); + remainingSize -= 4; + } + + if (remainingSize) return ERROR(srcSize_wrong); + return op-ostart; +} + + +size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, dict, dictSize); +#endif + ZSTD_decompressBegin_usingDict(dctx, dict, dictSize); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +size_t ZSTD_decompressDCtx(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_decompress_usingDict(dctx, dst, dstCapacity, src, srcSize, NULL, 0); +} + + +size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ +#if defined(ZSTD_HEAPMODE) && (ZSTD_HEAPMODE==1) + size_t regenSize; + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + if (dctx==NULL) return ERROR(memory_allocation); + regenSize = ZSTD_decompressDCtx(dctx, dst, dstCapacity, src, srcSize); + ZSTD_freeDCtx(dctx); + return regenSize; +#else /* stack mode */ + ZSTD_DCtx dctx; + return ZSTD_decompressDCtx(&dctx, dst, dstCapacity, src, srcSize); +#endif +} + + +/*-************************************** +* Advanced Streaming Decompression API +* Bufferless and synchronous +****************************************/ +size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } + +ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx) { + switch(dctx->stage) + { + default: /* should not happen */ + case ZSTDds_getFrameHeaderSize: + case ZSTDds_decodeFrameHeader: + return ZSTDnit_frameHeader; + case ZSTDds_decodeBlockHeader: + return ZSTDnit_blockHeader; + case ZSTDds_decompressBlock: + return ZSTDnit_block; + case ZSTDds_decompressLastBlock: + return ZSTDnit_lastBlock; + case ZSTDds_checkChecksum: + return ZSTDnit_checksum; + case ZSTDds_decodeSkippableHeader: + case ZSTDds_skipFrame: + return ZSTDnit_skippableFrame; + } +} + +int ZSTD_isSkipFrame(ZSTD_DCtx* dctx) { return dctx->stage == ZSTDds_skipFrame; } /* for zbuff */ + +/** ZSTD_decompressContinue() : +* @return : nb of bytes generated into `dst` (necessarily <= `dstCapacity) +* or an error code, which can be tested using ZSTD_isError() */ +size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + /* Sanity check */ + if (srcSize != dctx->expected) return ERROR(srcSize_wrong); + if (dstCapacity) ZSTD_checkContinuity(dctx, dst); + + switch (dctx->stage) + { + case ZSTDds_getFrameHeaderSize : + if (srcSize != ZSTD_frameHeaderSize_prefix) return ERROR(srcSize_wrong); /* impossible */ + if ((MEM_readLE32(src) & 0xFFFFFFF0U) == ZSTD_MAGIC_SKIPPABLE_START) { /* skippable frame */ + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); + dctx->expected = ZSTD_skippableHeaderSize - ZSTD_frameHeaderSize_prefix; /* magic number + skippable frame length */ + dctx->stage = ZSTDds_decodeSkippableHeader; + return 0; + } + dctx->headerSize = ZSTD_frameHeaderSize(src, ZSTD_frameHeaderSize_prefix); + if (ZSTD_isError(dctx->headerSize)) return dctx->headerSize; + memcpy(dctx->headerBuffer, src, ZSTD_frameHeaderSize_prefix); + if (dctx->headerSize > ZSTD_frameHeaderSize_prefix) { + dctx->expected = dctx->headerSize - ZSTD_frameHeaderSize_prefix; + dctx->stage = ZSTDds_decodeFrameHeader; + return 0; + } + dctx->expected = 0; /* not necessary to copy more */ + + case ZSTDds_decodeFrameHeader: + memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); + CHECK_F(ZSTD_decodeFrameHeader(dctx, dctx->headerBuffer, dctx->headerSize)); + dctx->expected = ZSTD_blockHeaderSize; + dctx->stage = ZSTDds_decodeBlockHeader; + return 0; + + case ZSTDds_decodeBlockHeader: + { blockProperties_t bp; + size_t const cBlockSize = ZSTD_getcBlockSize(src, ZSTD_blockHeaderSize, &bp); + if (ZSTD_isError(cBlockSize)) return cBlockSize; + dctx->expected = cBlockSize; + dctx->bType = bp.blockType; + dctx->rleSize = bp.origSize; + if (cBlockSize) { + dctx->stage = bp.lastBlock ? ZSTDds_decompressLastBlock : ZSTDds_decompressBlock; + return 0; + } + /* empty block */ + if (bp.lastBlock) { + if (dctx->fParams.checksumFlag) { + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* end of frame */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->expected = 3; /* go directly to next header */ + dctx->stage = ZSTDds_decodeBlockHeader; + } + return 0; + } + case ZSTDds_decompressLastBlock: + case ZSTDds_decompressBlock: + { size_t rSize; + switch(dctx->bType) + { + case bt_compressed: + rSize = ZSTD_decompressBlock_internal(dctx, dst, dstCapacity, src, srcSize); + break; + case bt_raw : + rSize = ZSTD_copyRawBlock(dst, dstCapacity, src, srcSize); + break; + case bt_rle : + rSize = ZSTD_setRleBlock(dst, dstCapacity, src, srcSize, dctx->rleSize); + break; + case bt_reserved : /* should never happen */ + default: + return ERROR(corruption_detected); + } + if (ZSTD_isError(rSize)) return rSize; + if (dctx->fParams.checksumFlag) XXH64_update(&dctx->xxhState, dst, rSize); + + if (dctx->stage == ZSTDds_decompressLastBlock) { /* end of frame */ + if (dctx->fParams.checksumFlag) { /* another round for frame checksum */ + dctx->expected = 4; + dctx->stage = ZSTDds_checkChecksum; + } else { + dctx->expected = 0; /* ends here */ + dctx->stage = ZSTDds_getFrameHeaderSize; + } + } else { + dctx->stage = ZSTDds_decodeBlockHeader; + dctx->expected = ZSTD_blockHeaderSize; + dctx->previousDstEnd = (char*)dst + rSize; + } + return rSize; + } + case ZSTDds_checkChecksum: + { U32 const h32 = (U32)XXH64_digest(&dctx->xxhState); + U32 const check32 = MEM_readLE32(src); /* srcSize == 4, guaranteed by dctx->expected */ + if (check32 != h32) return ERROR(checksum_wrong); + dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + case ZSTDds_decodeSkippableHeader: + { memcpy(dctx->headerBuffer + ZSTD_frameHeaderSize_prefix, src, dctx->expected); + dctx->expected = MEM_readLE32(dctx->headerBuffer + 4); + dctx->stage = ZSTDds_skipFrame; + return 0; + } + case ZSTDds_skipFrame: + { dctx->expected = 0; + dctx->stage = ZSTDds_getFrameHeaderSize; + return 0; + } + default: + return ERROR(GENERIC); /* impossible */ + } +} + + +static size_t ZSTD_refDictContent(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + dctx->dictEnd = dctx->previousDstEnd; + dctx->vBase = (const char*)dict - ((const char*)(dctx->previousDstEnd) - (const char*)(dctx->base)); + dctx->base = dict; + dctx->previousDstEnd = (const char*)dict + dictSize; + return 0; +} + +static size_t ZSTD_loadEntropy(ZSTD_DCtx* dctx, const void* const dict, size_t const dictSize) +{ + const BYTE* dictPtr = (const BYTE*)dict; + const BYTE* const dictEnd = dictPtr + dictSize; + + { size_t const hSize = HUF_readDTableX4(dctx->hufTable, dict, dictSize); + if (HUF_isError(hSize)) return ERROR(dictionary_corrupted); + dictPtr += hSize; + } + + { short offcodeNCount[MaxOff+1]; + U32 offcodeMaxValue=MaxOff, offcodeLog; + size_t const offcodeHeaderSize = FSE_readNCount(offcodeNCount, &offcodeMaxValue, &offcodeLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(offcodeHeaderSize)) return ERROR(dictionary_corrupted); + if (offcodeLog > OffFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->OFTable, offcodeNCount, offcodeMaxValue, offcodeLog), dictionary_corrupted); + dictPtr += offcodeHeaderSize; + } + + { short matchlengthNCount[MaxML+1]; + unsigned matchlengthMaxValue = MaxML, matchlengthLog; + size_t const matchlengthHeaderSize = FSE_readNCount(matchlengthNCount, &matchlengthMaxValue, &matchlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(matchlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (matchlengthLog > MLFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->MLTable, matchlengthNCount, matchlengthMaxValue, matchlengthLog), dictionary_corrupted); + dictPtr += matchlengthHeaderSize; + } + + { short litlengthNCount[MaxLL+1]; + unsigned litlengthMaxValue = MaxLL, litlengthLog; + size_t const litlengthHeaderSize = FSE_readNCount(litlengthNCount, &litlengthMaxValue, &litlengthLog, dictPtr, dictEnd-dictPtr); + if (FSE_isError(litlengthHeaderSize)) return ERROR(dictionary_corrupted); + if (litlengthLog > LLFSELog) return ERROR(dictionary_corrupted); + CHECK_E(FSE_buildDTable(dctx->LLTable, litlengthNCount, litlengthMaxValue, litlengthLog), dictionary_corrupted); + dictPtr += litlengthHeaderSize; + } + + if (dictPtr+12 > dictEnd) return ERROR(dictionary_corrupted); + dctx->rep[0] = MEM_readLE32(dictPtr+0); if (dctx->rep[0] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[1] = MEM_readLE32(dictPtr+4); if (dctx->rep[1] >= dictSize) return ERROR(dictionary_corrupted); + dctx->rep[2] = MEM_readLE32(dictPtr+8); if (dctx->rep[2] >= dictSize) return ERROR(dictionary_corrupted); + dictPtr += 12; + + dctx->litEntropy = dctx->fseEntropy = 1; + return dictPtr - (const BYTE*)dict; +} + +static size_t ZSTD_decompress_insertDictionary(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + if (dictSize < 8) return ZSTD_refDictContent(dctx, dict, dictSize); + { U32 const magic = MEM_readLE32(dict); + if (magic != ZSTD_DICT_MAGIC) { + return ZSTD_refDictContent(dctx, dict, dictSize); /* pure content mode */ + } } + dctx->dictID = MEM_readLE32((const char*)dict + 4); + + /* load entropy tables */ + dict = (const char*)dict + 8; + dictSize -= 8; + { size_t const eSize = ZSTD_loadEntropy(dctx, dict, dictSize); + if (ZSTD_isError(eSize)) return ERROR(dictionary_corrupted); + dict = (const char*)dict + eSize; + dictSize -= eSize; + } + + /* reference dictionary content */ + return ZSTD_refDictContent(dctx, dict, dictSize); +} + +size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize) +{ + CHECK_F(ZSTD_decompressBegin(dctx)); + if (dict && dictSize) CHECK_E(ZSTD_decompress_insertDictionary(dctx, dict, dictSize), dictionary_corrupted); + return 0; +} + + +/* ====== ZSTD_DDict ====== */ + +struct ZSTD_DDict_s { + void* dict; + size_t dictSize; + ZSTD_DCtx* refContext; +}; /* typedef'd to ZSTD_DDict within "zstd.h" */ + +ZSTD_DDict* ZSTD_createDDict_advanced(const void* dict, size_t dictSize, ZSTD_customMem customMem) +{ + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + { ZSTD_DDict* const ddict = (ZSTD_DDict*) ZSTD_malloc(sizeof(ZSTD_DDict), customMem); + void* const dictContent = ZSTD_malloc(dictSize, customMem); + ZSTD_DCtx* const dctx = ZSTD_createDCtx_advanced(customMem); + + if (!dictContent || !ddict || !dctx) { + ZSTD_free(dictContent, customMem); + ZSTD_free(ddict, customMem); + ZSTD_free(dctx, customMem); + return NULL; + } + + if (dictSize) { + memcpy(dictContent, dict, dictSize); + } + { size_t const errorCode = ZSTD_decompressBegin_usingDict(dctx, dictContent, dictSize); + if (ZSTD_isError(errorCode)) { + ZSTD_free(dictContent, customMem); + ZSTD_free(ddict, customMem); + ZSTD_free(dctx, customMem); + return NULL; + } } + + ddict->dict = dictContent; + ddict->dictSize = dictSize; + ddict->refContext = dctx; + return ddict; + } +} + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression without startup delay. +* `dict` can be released after `ZSTD_DDict` creation */ +ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize) +{ + ZSTD_customMem const allocator = { NULL, NULL, NULL }; + return ZSTD_createDDict_advanced(dict, dictSize, allocator); +} + +size_t ZSTD_freeDDict(ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support free on NULL */ + { ZSTD_customMem const cMem = ddict->refContext->customMem; + ZSTD_freeDCtx(ddict->refContext); + ZSTD_free(ddict->dict, cMem); + ZSTD_free(ddict, cMem); + return 0; + } +} + +size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*ddict) + sizeof(ddict->refContext) + ddict->dictSize; +} + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dict) != ZSTD_DICT_MAGIC) return 0; + return MEM_readLE32((const char*)dict + 4); +} + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) +{ + if (ddict==NULL) return 0; + return ZSTD_getDictID_fromDict(ddict->dict, ddict->dictSize); +} + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) +{ + ZSTD_frameParams zfp = { 0 , 0 , 0 , 0 }; + size_t const hError = ZSTD_getFrameParams(&zfp, src, srcSize); + if (ZSTD_isError(hError)) return 0; + return zfp.dictID; +} + + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a pre-digested Dictionary +* Use dictionary without significant overhead. */ +size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict) +{ +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT==1) + if (ZSTD_isLegacy(src, srcSize)) return ZSTD_decompressLegacy(dst, dstCapacity, src, srcSize, ddict->dict, ddict->dictSize); +#endif + ZSTD_refDCtx(dctx, ddict->refContext); + ZSTD_checkContinuity(dctx, dst); + return ZSTD_decompressFrame(dctx, dst, dstCapacity, src, srcSize); +} + + +/*===================================== +* Streaming decompression +*====================================*/ + +typedef enum { zdss_init, zdss_loadHeader, + zdss_read, zdss_load, zdss_flush } ZSTD_dStreamStage; + +/* *** Resource management *** */ +struct ZSTD_DStream_s { + ZSTD_DCtx* dctx; + ZSTD_DDict* ddictLocal; + const ZSTD_DDict* ddict; + ZSTD_frameParams fParams; + ZSTD_dStreamStage stage; + char* inBuff; + size_t inBuffSize; + size_t inPos; + size_t maxWindowSize; + char* outBuff; + size_t outBuffSize; + size_t outStart; + size_t outEnd; + size_t blockSize; + BYTE headerBuffer[ZSTD_FRAMEHEADERSIZE_MAX]; /* tmp buffer to store frame header */ + size_t lhSize; + ZSTD_customMem customMem; + void* legacyContext; + U32 previousLegacyVersion; + U32 legacyVersion; + U32 hostageByte; +}; /* typedef'd to ZSTD_DStream within "zstd.h" */ + + +ZSTD_DStream* ZSTD_createDStream(void) +{ + return ZSTD_createDStream_advanced(defaultCustomMem); +} + +ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem) +{ + ZSTD_DStream* zds; + + if (!customMem.customAlloc && !customMem.customFree) customMem = defaultCustomMem; + if (!customMem.customAlloc || !customMem.customFree) return NULL; + + zds = (ZSTD_DStream*) ZSTD_malloc(sizeof(ZSTD_DStream), customMem); + if (zds==NULL) return NULL; + memset(zds, 0, sizeof(ZSTD_DStream)); + memcpy(&zds->customMem, &customMem, sizeof(ZSTD_customMem)); + zds->dctx = ZSTD_createDCtx_advanced(customMem); + if (zds->dctx == NULL) { ZSTD_freeDStream(zds); return NULL; } + zds->stage = zdss_init; + zds->maxWindowSize = ZSTD_MAXWINDOWSIZE_DEFAULT; + return zds; +} + +size_t ZSTD_freeDStream(ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support free on null */ + { ZSTD_customMem const cMem = zds->customMem; + ZSTD_freeDCtx(zds->dctx); + ZSTD_freeDDict(zds->ddictLocal); + ZSTD_free(zds->inBuff, cMem); + ZSTD_free(zds->outBuff, cMem); +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT >= 1) + if (zds->legacyContext) + ZSTD_freeLegacyStreamContext(zds->legacyContext, zds->previousLegacyVersion); +#endif + ZSTD_free(zds, cMem); + return 0; + } +} + + +/* *** Initialization *** */ + +size_t ZSTD_DStreamInSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX + ZSTD_blockHeaderSize; } +size_t ZSTD_DStreamOutSize(void) { return ZSTD_BLOCKSIZE_ABSOLUTEMAX; } + +size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + ZSTD_freeDDict(zds->ddictLocal); + if (dict) { + zds->ddictLocal = ZSTD_createDDict(dict, dictSize); + if (zds->ddictLocal == NULL) return ERROR(memory_allocation); + } else zds->ddictLocal = NULL; + zds->ddict = zds->ddictLocal; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + +size_t ZSTD_initDStream(ZSTD_DStream* zds) +{ + return ZSTD_initDStream_usingDict(zds, NULL, 0); +} + +size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict) /**< note : ddict will just be referenced, and must outlive decompression session */ +{ + size_t const initResult = ZSTD_initDStream(zds); + zds->ddict = ddict; + return initResult; +} + +size_t ZSTD_resetDStream(ZSTD_DStream* zds) +{ + zds->stage = zdss_loadHeader; + zds->lhSize = zds->inPos = zds->outStart = zds->outEnd = 0; + zds->legacyVersion = 0; + zds->hostageByte = 0; + return ZSTD_frameHeaderSize_prefix; +} + +size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, + ZSTD_DStreamParameter_e paramType, unsigned paramValue) +{ + switch(paramType) + { + default : return ERROR(parameter_unknown); + case ZSTDdsp_maxWindowSize : zds->maxWindowSize = paramValue ? paramValue : (U32)(-1); break; + } + return 0; +} + + +size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds) +{ + if (zds==NULL) return 0; /* support sizeof on NULL */ + return sizeof(*zds) + ZSTD_sizeof_DCtx(zds->dctx) + ZSTD_sizeof_DDict(zds->ddictLocal) + zds->inBuffSize + zds->outBuffSize; +} + + +/* ***** Decompression ***** */ + +MEM_STATIC size_t ZSTD_limitCopy(void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + size_t const length = MIN(dstCapacity, srcSize); + memcpy(dst, src, length); + return length; +} + + +size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input) +{ + const char* const istart = (const char*)(input->src) + input->pos; + const char* const iend = (const char*)(input->src) + input->size; + const char* ip = istart; + char* const ostart = (char*)(output->dst) + output->pos; + char* const oend = (char*)(output->dst) + output->size; + char* op = ostart; + U32 someMoreWork = 1; + +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + if (zds->legacyVersion) + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); +#endif + + while (someMoreWork) { + switch(zds->stage) + { + case zdss_init : + ZSTD_resetDStream(zds); /* transparent reset on starting decoding a new frame */ + /* fall-through */ + + case zdss_loadHeader : + { size_t const hSize = ZSTD_getFrameParams(&zds->fParams, zds->headerBuffer, zds->lhSize); + if (ZSTD_isError(hSize)) +#if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) + { U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); + if (legacyVersion) { + const void* const dict = zds->ddict ? zds->ddict->dict : NULL; + size_t const dictSize = zds->ddict ? zds->ddict->dictSize : 0; + CHECK_F(ZSTD_initLegacyStream(&zds->legacyContext, zds->previousLegacyVersion, legacyVersion, + dict, dictSize)); + zds->legacyVersion = zds->previousLegacyVersion = legacyVersion; + return ZSTD_decompressLegacyStream(zds->legacyContext, zds->legacyVersion, output, input); + } else { + return hSize; /* error */ + } } +#else + return hSize; +#endif + if (hSize != 0) { /* need more input */ + size_t const toLoad = hSize - zds->lhSize; /* if hSize!=0, hSize > zds->lhSize */ + if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ + memcpy(zds->headerBuffer + zds->lhSize, ip, iend-ip); + zds->lhSize += iend-ip; + input->pos = input->size; + return (MAX(ZSTD_frameHeaderSize_min, hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ + } + memcpy(zds->headerBuffer + zds->lhSize, ip, toLoad); zds->lhSize = hSize; ip += toLoad; + break; + } } + + /* Consume header */ + { const ZSTD_DCtx* refContext = zds->ddict ? zds->ddict->refContext : NULL; + ZSTD_refDCtx(zds->dctx, refContext); + } + { size_t const h1Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); /* == ZSTD_frameHeaderSize_prefix */ + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer, h1Size)); + { size_t const h2Size = ZSTD_nextSrcSizeToDecompress(zds->dctx); + CHECK_F(ZSTD_decompressContinue(zds->dctx, NULL, 0, zds->headerBuffer+h1Size, h2Size)); + } } + + zds->fParams.windowSize = MAX(zds->fParams.windowSize, 1U << ZSTD_WINDOWLOG_ABSOLUTEMIN); + if (zds->fParams.windowSize > zds->maxWindowSize) return ERROR(frameParameter_windowTooLarge); + + /* Adapt buffer sizes to frame header instructions */ + { size_t const blockSize = MIN(zds->fParams.windowSize, ZSTD_BLOCKSIZE_ABSOLUTEMAX); + size_t const neededOutSize = zds->fParams.windowSize + blockSize; + zds->blockSize = blockSize; + if (zds->inBuffSize < blockSize) { + ZSTD_free(zds->inBuff, zds->customMem); + zds->inBuffSize = blockSize; + zds->inBuff = (char*)ZSTD_malloc(blockSize, zds->customMem); + if (zds->inBuff == NULL) return ERROR(memory_allocation); + } + if (zds->outBuffSize < neededOutSize) { + ZSTD_free(zds->outBuff, zds->customMem); + zds->outBuffSize = neededOutSize; + zds->outBuff = (char*)ZSTD_malloc(neededOutSize, zds->customMem); + if (zds->outBuff == NULL) return ERROR(memory_allocation); + } } + zds->stage = zdss_read; + /* pass-through */ + + case zdss_read: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); + if (neededInSize==0) { /* end of frame */ + zds->stage = zdss_init; + someMoreWork = 0; + break; + } + if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ + const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, + zds->outBuff + zds->outStart, (isSkipFrame ? 0 : zds->outBuffSize - zds->outStart), + ip, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + ip += neededInSize; + if (!decodedSize && !isSkipFrame) break; /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + break; + } + if (ip==iend) { someMoreWork = 0; break; } /* no more input */ + zds->stage = zdss_load; + /* pass-through */ + } + + case zdss_load: + { size_t const neededInSize = ZSTD_nextSrcSizeToDecompress(zds->dctx); + size_t const toLoad = neededInSize - zds->inPos; /* should always be <= remaining space within inBuff */ + size_t loadedSize; + if (toLoad > zds->inBuffSize - zds->inPos) return ERROR(corruption_detected); /* should never happen */ + loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, iend-ip); + ip += loadedSize; + zds->inPos += loadedSize; + if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ + + /* decode loaded input */ + { const int isSkipFrame = ZSTD_isSkipFrame(zds->dctx); + size_t const decodedSize = ZSTD_decompressContinue(zds->dctx, + zds->outBuff + zds->outStart, zds->outBuffSize - zds->outStart, + zds->inBuff, neededInSize); + if (ZSTD_isError(decodedSize)) return decodedSize; + zds->inPos = 0; /* input is consumed */ + if (!decodedSize && !isSkipFrame) { zds->stage = zdss_read; break; } /* this was just a header */ + zds->outEnd = zds->outStart + decodedSize; + zds->stage = zdss_flush; + /* pass-through */ + } } + + case zdss_flush: + { size_t const toFlushSize = zds->outEnd - zds->outStart; + size_t const flushedSize = ZSTD_limitCopy(op, oend-op, zds->outBuff + zds->outStart, toFlushSize); + op += flushedSize; + zds->outStart += flushedSize; + if (flushedSize == toFlushSize) { /* flush completed */ + zds->stage = zdss_read; + if (zds->outStart + zds->blockSize > zds->outBuffSize) + zds->outStart = zds->outEnd = 0; + break; + } + /* cannot complete flush */ + someMoreWork = 0; + break; + } + default: return ERROR(GENERIC); /* impossible */ + } } + + /* result */ + input->pos += (size_t)(ip-istart); + output->pos += (size_t)(op-ostart); + { size_t nextSrcSizeHint = ZSTD_nextSrcSizeToDecompress(zds->dctx); + if (!nextSrcSizeHint) { /* frame fully decoded */ + if (zds->outEnd == zds->outStart) { /* output fully flushed */ + if (zds->hostageByte) { + if (input->pos >= input->size) { zds->stage = zdss_read; return 1; } /* can't release hostage (not present) */ + input->pos++; /* release hostage */ + } + return 0; + } + if (!zds->hostageByte) { /* output not fully flushed; keep last byte as hostage; will be released when all output is flushed */ + input->pos--; /* note : pos > 0, otherwise, impossible to finish reading last block */ + zds->hostageByte=1; + } + return 1; + } + nextSrcSizeHint += ZSTD_blockHeaderSize * (ZSTD_nextInputType(zds->dctx) == ZSTDnit_block); /* preload header of next block */ + if (zds->inPos > nextSrcSizeHint) return ERROR(GENERIC); /* should never happen */ + nextSrcSizeHint -= zds->inPos; /* already loaded*/ + return nextSrcSizeHint; + } +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,1913 @@ +/* + * divsufsort.c for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +/*- Compiler specifics -*/ +#ifdef __clang__ +#pragma clang diagnostic ignored "-Wshorten-64-to-32" +#endif + +#if defined(_MSC_VER) +# pragma warning(disable : 4244) +# pragma warning(disable : 4127) /* C4127 : Condition expression is constant */ +#endif + + +/*- Dependencies -*/ +#include +#include +#include + +#include "divsufsort.h" + +/*- Constants -*/ +#if defined(INLINE) +# undef INLINE +#endif +#if !defined(INLINE) +# define INLINE __inline +#endif +#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) +# undef ALPHABET_SIZE +#endif +#if !defined(ALPHABET_SIZE) +# define ALPHABET_SIZE (256) +#endif +#define BUCKET_A_SIZE (ALPHABET_SIZE) +#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) +#if defined(SS_INSERTIONSORT_THRESHOLD) +# if SS_INSERTIONSORT_THRESHOLD < 1 +# undef SS_INSERTIONSORT_THRESHOLD +# define SS_INSERTIONSORT_THRESHOLD (1) +# endif +#else +# define SS_INSERTIONSORT_THRESHOLD (8) +#endif +#if defined(SS_BLOCKSIZE) +# if SS_BLOCKSIZE < 0 +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (0) +# elif 32768 <= SS_BLOCKSIZE +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (32767) +# endif +#else +# define SS_BLOCKSIZE (1024) +#endif +/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ +#if SS_BLOCKSIZE == 0 +# define SS_MISORT_STACKSIZE (96) +#elif SS_BLOCKSIZE <= 4096 +# define SS_MISORT_STACKSIZE (16) +#else +# define SS_MISORT_STACKSIZE (24) +#endif +#define SS_SMERGE_STACKSIZE (32) +#define TR_INSERTIONSORT_THRESHOLD (8) +#define TR_STACKSIZE (64) + + +/*- Macros -*/ +#ifndef SWAP +# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) +#endif /* SWAP */ +#ifndef MIN +# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) +#endif /* MIN */ +#ifndef MAX +# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) +#endif /* MAX */ +#define STACK_PUSH(_a, _b, _c, _d)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize++].d = (_d);\ + } while(0) +#define STACK_PUSH5(_a, _b, _c, _d, _e)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ + } while(0) +#define STACK_POP(_a, _b, _c, _d)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ + } while(0) +#define STACK_POP5(_a, _b, _c, _d, _e)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ + } while(0) +#define BUCKET_A(_c0) bucket_A[(_c0)] +#if ALPHABET_SIZE == 256 +#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) +#else +#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) +#endif + + +/*- Private Functions -*/ + +static const int lg_table[256]= { + -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 +}; + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +int +ss_ilg(int n) { +#if SS_BLOCKSIZE == 0 + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +#elif SS_BLOCKSIZE < 256 + return lg_table[n]; +#else + return (n & 0xff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]; +#endif +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + +#if SS_BLOCKSIZE != 0 + +static const int sqq_table[256] = { + 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61, + 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89, + 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109, +110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, +128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155, +156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168, +169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180, +181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191, +192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201, +202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211, +212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221, +221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230, +230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238, +239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, +247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255 +}; + +static INLINE +int +ss_isqrt(int x) { + int y, e; + + if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; } + e = (x & 0xffff0000) ? + ((x & 0xff000000) ? + 24 + lg_table[(x >> 24) & 0xff] : + 16 + lg_table[(x >> 16) & 0xff]) : + ((x & 0x0000ff00) ? + 8 + lg_table[(x >> 8) & 0xff] : + 0 + lg_table[(x >> 0) & 0xff]); + + if(e >= 16) { + y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7); + if(e >= 24) { y = (y + 1 + x / y) >> 1; } + y = (y + 1 + x / y) >> 1; + } else if(e >= 8) { + y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1; + } else { + return sqq_table[x] >> 4; + } + + return (x < (y * y)) ? y - 1 : y; +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Compares two suffixes. */ +static INLINE +int +ss_compare(const unsigned char *T, + const int *p1, const int *p2, + int depth) { + const unsigned char *U1, *U2, *U1n, *U2n; + + for(U1 = T + depth + *p1, + U2 = T + depth + *p2, + U1n = T + *(p1 + 1) + 2, + U2n = T + *(p2 + 1) + 2; + (U1 < U1n) && (U2 < U2n) && (*U1 == *U2); + ++U1, ++U2) { + } + + return U1 < U1n ? + (U2 < U2n ? *U1 - *U2 : 1) : + (U2 < U2n ? -1 : 0); +} + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) + +/* Insertionsort for small size groups */ +static +void +ss_insertionsort(const unsigned char *T, const int *PA, + int *first, int *last, int depth) { + int *i, *j; + int t; + int r; + + for(i = last - 2; first <= i; --i) { + for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) { + do { *(j - 1) = *j; } while((++j < last) && (*j < 0)); + if(last <= j) { break; } + } + if(r == 0) { *j = ~*j; } + *(j - 1) = t; + } +} + +#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */ + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +void +ss_fixdown(const unsigned char *Td, const int *PA, + int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = Td[PA[SA[k = j++]]]; + if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +ss_heapsort(const unsigned char *Td, const int *PA, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + ss_fixdown(Td, PA, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +ss_median3(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3) { + int *t; + if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); } + if(Td[PA[*v2]] > Td[PA[*v3]]) { + if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +ss_median5(const unsigned char *Td, const int *PA, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); } + if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); } + if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); } + if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); } + if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); } + if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +ss_pivot(const unsigned char *Td, const int *PA, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return ss_median3(Td, PA, first, middle, last - 1); + } else { + t >>= 2; + return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = ss_median3(Td, PA, first, first + t, first + (t << 1)); + middle = ss_median3(Td, PA, middle - t, middle, middle + t); + last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1); + return ss_median3(Td, PA, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +/* Binary partition for substrings. */ +static INLINE +int * +ss_partition(const int *PA, + int *first, int *last, int depth) { + int *a, *b; + int t; + for(a = first - 1, b = last;;) { + for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; } + for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { } + if(b <= a) { break; } + t = ~*b; + *b = *a; + *a = t; + } + if(first < a) { *first = ~*first; } + return a; +} + +/* Multikey introsort for medium size groups. */ +static +void +ss_mintrosort(const unsigned char *T, const int *PA, + int *first, int *last, + int depth) { +#define STACK_SIZE SS_MISORT_STACKSIZE + struct { int *a, *b, c; int d; } stack[STACK_SIZE]; + const unsigned char *Td; + int *a, *b, *c, *d, *e, *f; + int s, t; + int ssize; + int limit; + int v, x = 0; + + for(ssize = 0, limit = ss_ilg(last - first);;) { + + if((last - first) <= SS_INSERTIONSORT_THRESHOLD) { +#if 1 < SS_INSERTIONSORT_THRESHOLD + if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); } +#endif + STACK_POP(first, last, depth, limit); + continue; + } + + Td = T + depth; + if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); } + if(limit < 0) { + for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) { + if((x = Td[PA[*a]]) != v) { + if(1 < (a - first)) { break; } + v = x; + first = a; + } + } + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, a, depth); + } + if((a - first) <= (last - a)) { + if(1 < (a - first)) { + STACK_PUSH(a, last, depth, -1); + last = a, depth += 1, limit = ss_ilg(a - first); + } else { + first = a, limit = -1; + } + } else { + if(1 < (last - a)) { + STACK_PUSH(first, a, depth + 1, ss_ilg(a - first)); + first = a, limit = -1; + } else { + last = a, depth += 1, limit = ss_ilg(a - first); + } + } + continue; + } + + /* choose pivot */ + a = ss_pivot(Td, PA, first, last); + v = Td[PA[*a]]; + SWAP(*first, *a); + + /* partition */ + for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + + a = first + (b - a), c = last - (d - c); + b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth); + + if((a - first) <= (last - c)) { + if((last - c) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(c, last, depth, limit); + last = a; + } else if((a - first) <= (c - b)) { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + last = a; + } else { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(first, a, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } else { + if((a - first) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(first, a, depth, limit); + first = c; + } else if((last - c) <= (c - b)) { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + first = c; + } else { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(c, last, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } + } else { + limit += 1; + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, last, depth); + limit = ss_ilg(last - first); + } + depth += 1; + } + } +#undef STACK_SIZE +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + + +/*---------------------------------------------------------------------------*/ + +#if SS_BLOCKSIZE != 0 + +static INLINE +void +ss_blockswap(int *a, int *b, int n) { + int t; + for(; 0 < n; --n, ++a, ++b) { + t = *a, *a = *b, *b = t; + } +} + +static INLINE +void +ss_rotate(int *first, int *middle, int *last) { + int *a, *b, t; + int l, r; + l = middle - first, r = last - middle; + for(; (0 < l) && (0 < r);) { + if(l == r) { ss_blockswap(first, middle, l); break; } + if(l < r) { + a = last - 1, b = middle - 1; + t = *a; + do { + *a-- = *b, *b-- = *a; + if(b < first) { + *a = t; + last = a; + if((r -= l + 1) <= l) { break; } + a -= 1, b = middle - 1; + t = *a; + } + } while(1); + } else { + a = first, b = middle; + t = *a; + do { + *a++ = *b, *b++ = *a; + if(last <= b) { + *a = t; + first = a + 1; + if((l -= r + 1) <= r) { break; } + a += 1, b = middle; + t = *a; + } + } while(1); + } + } +} + + +/*---------------------------------------------------------------------------*/ + +static +void +ss_inplacemerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int depth) { + const int *p; + int *a, *b; + int len, half; + int q, r; + int x; + + for(;;) { + if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); } + else { x = 0; p = PA + *(last - 1); } + for(a = first, len = middle - first, half = len >> 1, r = -1; + 0 < len; + len = half, half >>= 1) { + b = a + half; + q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth); + if(q < 0) { + a = b + 1; + half -= (len & 1) ^ 1; + } else { + r = q; + } + } + if(a < middle) { + if(r == 0) { *a = ~*a; } + ss_rotate(a, middle, last); + last -= middle - a; + middle = a; + if(first == middle) { break; } + } + --last; + if(x != 0) { while(*--last < 0) { } } + if(middle == last) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Merge-forward with internal buffer. */ +static +void +ss_mergeforward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + int *a, *b, *c, *bufend; + int t; + int r; + + bufend = buf + (middle - first) - 1; + ss_blockswap(buf, first, middle - first); + + for(t = *(a = first), b = buf, c = middle;;) { + r = ss_compare(T, PA + *b, PA + *c, depth); + if(r < 0) { + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + } else if(r > 0) { + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } else { + *c = ~*c; + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } + } +} + +/* Merge-backward with internal buffer. */ +static +void +ss_mergebackward(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int depth) { + const int *p1, *p2; + int *a, *b, *c, *bufend; + int t; + int r; + int x; + + bufend = buf + (last - middle) - 1; + ss_blockswap(buf, middle, last - middle); + + x = 0; + if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; } + else { p1 = PA + *bufend; } + if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; } + else { p2 = PA + *(middle - 1); } + for(t = *(a = last - 1), b = bufend, c = middle - 1;;) { + r = ss_compare(T, p1, p2, depth); + if(0 < r) { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = *b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + } else if(r < 0) { + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } else { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = ~*b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } + } +} + +/* D&C based merge. */ +static +void +ss_swapmerge(const unsigned char *T, const int *PA, + int *first, int *middle, int *last, + int *buf, int bufsize, int depth) { +#define STACK_SIZE SS_SMERGE_STACKSIZE +#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a))) +#define MERGE_CHECK(a, b, c)\ + do {\ + if(((c) & 1) ||\ + (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\ + *(a) = ~*(a);\ + }\ + if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\ + *(b) = ~*(b);\ + }\ + } while(0) + struct { int *a, *b, *c; int d; } stack[STACK_SIZE]; + int *l, *r, *lm, *rm; + int m, len, half; + int ssize; + int check, next; + + for(check = 0, ssize = 0;;) { + if((last - middle) <= bufsize) { + if((first < middle) && (middle < last)) { + ss_mergebackward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + if((middle - first) <= bufsize) { + if(first < middle) { + ss_mergeforward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1) { + if(ss_compare(T, PA + GETIDX(*(middle + m + half)), + PA + GETIDX(*(middle - m - half - 1)), depth) < 0) { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if(0 < m) { + lm = middle - m, rm = middle + m; + ss_blockswap(lm, middle, m); + l = r = middle, next = 0; + if(rm < last) { + if(*rm < 0) { + *rm = ~*rm; + if(first < lm) { for(; *--l < 0;) { } next |= 4; } + next |= 1; + } else if(first < lm) { + for(; *r < 0; ++r) { } + next |= 2; + } + } + + if((l - first) <= (last - r)) { + STACK_PUSH(r, rm, last, (next & 3) | (check & 4)); + middle = lm, last = l, check = (check & 3) | (next & 4); + } else { + if((next & 2) && (r == middle)) { next ^= 6; } + STACK_PUSH(first, lm, l, (check & 3) | (next & 4)); + first = r, middle = rm, check = (next & 3) | (check & 4); + } + } else { + if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) { + *middle = ~*middle; + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + } + } +#undef STACK_SIZE +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Substring sort */ +static +void +sssort(const unsigned char *T, const int *PA, + int *first, int *last, + int *buf, int bufsize, + int depth, int n, int lastsuffix) { + int *a; +#if SS_BLOCKSIZE != 0 + int *b, *middle, *curbuf; + int j, k, curbufsize, limit; +#endif + int i; + + if(lastsuffix != 0) { ++first; } + +#if SS_BLOCKSIZE == 0 + ss_mintrosort(T, PA, first, last, depth); +#else + if((bufsize < SS_BLOCKSIZE) && + (bufsize < (last - first)) && + (bufsize < (limit = ss_isqrt(last - first)))) { + if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; } + buf = middle = last - limit, bufsize = limit; + } else { + middle = last, limit = 0; + } + for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth); +#endif + curbufsize = last - (a + SS_BLOCKSIZE); + curbuf = a + SS_BLOCKSIZE; + if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; } + for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) { + ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth); + } + } +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, middle, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, middle, depth); +#endif + for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { + if(i & 1) { + ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth); + a -= k; + } + } + if(limit != 0) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, middle, last, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, middle, last, depth); +#endif + ss_inplacemerge(T, PA, first, middle, last, depth); + } +#endif + + if(lastsuffix != 0) { + /* Insert last type B* suffix. */ + int PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2; + for(a = first, i = *(first - 1); + (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth))); + ++a) { + *(a - 1) = *a; + } + *(a - 1) = i; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +int +tr_ilg(int n) { + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +} + + +/*---------------------------------------------------------------------------*/ + +/* Simple insertionsort for small size groups. */ +static +void +tr_insertionsort(const int *ISAd, int *first, int *last) { + int *a, *b; + int t, r; + + for(a = first + 1; a < last; ++a) { + for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) { + do { *(b + 1) = *b; } while((first <= --b) && (*b < 0)); + if(b < first) { break; } + } + if(r == 0) { *b = ~*b; } + *(b + 1) = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_fixdown(const int *ISAd, int *SA, int i, int size) { + int j, k; + int v; + int c, d, e; + + for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = ISAd[SA[k = j++]]; + if(d < (e = ISAd[SA[j]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +tr_heapsort(const int *ISAd, int *SA, int size) { + int i, m; + int t; + + m = size; + if((size % 2) == 0) { + m--; + if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + tr_fixdown(ISAd, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +int * +tr_median3(const int *ISAd, int *v1, int *v2, int *v3) { + int *t; + if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); } + if(ISAd[*v2] > ISAd[*v3]) { + if(ISAd[*v1] > ISAd[*v3]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +int * +tr_median5(const int *ISAd, + int *v1, int *v2, int *v3, int *v4, int *v5) { + int *t; + if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); } + if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); } + if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); } + if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); } + if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); } + if(ISAd[*v3] > ISAd[*v4]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +int * +tr_pivot(const int *ISAd, int *first, int *last) { + int *middle; + int t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return tr_median3(ISAd, first, middle, last - 1); + } else { + t >>= 2; + return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = tr_median3(ISAd, first, first + t, first + (t << 1)); + middle = tr_median3(ISAd, middle - t, middle, middle + t); + last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1); + return tr_median3(ISAd, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +typedef struct _trbudget_t trbudget_t; +struct _trbudget_t { + int chance; + int remain; + int incval; + int count; +}; + +static INLINE +void +trbudget_init(trbudget_t *budget, int chance, int incval) { + budget->chance = chance; + budget->remain = budget->incval = incval; +} + +static INLINE +int +trbudget_check(trbudget_t *budget, int size) { + if(size <= budget->remain) { budget->remain -= size; return 1; } + if(budget->chance == 0) { budget->count += size; return 0; } + budget->remain += budget->incval - size; + budget->chance -= 1; + return 1; +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_partition(const int *ISAd, + int *first, int *middle, int *last, + int **pa, int **pb, int v) { + int *a, *b, *c, *d, *e, *f; + int t, s; + int x = 0; + + for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + first += (b - a), last -= (d - c); + } + *pa = first, *pb = last; +} + +static +void +tr_copy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + /* sort suffixes of middle partition + by using sorted order of suffixes of left and right partition. */ + int *c, *d, *e; + int s, v; + + v = b - SA - 1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + ISA[s] = d - SA; + } + } + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + ISA[s] = d - SA; + } + } +} + +static +void +tr_partialcopy(int *ISA, const int *SA, + int *first, int *a, int *b, int *last, + int depth) { + int *c, *d, *e; + int s, v; + int rank, lastrank, newrank = -1; + + v = b - SA - 1; + lastrank = -1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } + + lastrank = -1; + for(e = d; first <= e; --e) { + rank = ISA[*e]; + if(lastrank != rank) { lastrank = rank; newrank = e - SA; } + if(newrank != rank) { ISA[*e] = newrank; } + } + + lastrank = -1; + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } +} + +static +void +tr_introsort(int *ISA, const int *ISAd, + int *SA, int *first, int *last, + trbudget_t *budget) { +#define STACK_SIZE TR_STACKSIZE + struct { const int *a; int *b, *c; int d, e; }stack[STACK_SIZE]; + int *a, *b, *c; + int t; + int v, x = 0; + int incr = ISAd - ISA; + int limit, next; + int ssize, trlink = -1; + + for(ssize = 0, limit = tr_ilg(last - first);;) { + + if(limit < 0) { + if(limit == -1) { + /* tandem repeat partition */ + tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1); + + /* update ranks */ + if(a < last) { + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + } + if(b < last) { + for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } + } + + /* push */ + if(1 < (b - a)) { + STACK_PUSH5(NULL, a, b, 0, 0); + STACK_PUSH5(ISAd - incr, first, last, -2, trlink); + trlink = ssize - 2; + } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink); + last = a, limit = tr_ilg(a - first); + } else if(1 < (last - b)) { + first = b, limit = tr_ilg(last - b); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink); + first = b, limit = tr_ilg(last - b); + } else if(1 < (a - first)) { + last = a, limit = tr_ilg(a - first); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else if(limit == -2) { + /* tandem repeat copy */ + a = stack[--ssize].b, b = stack[ssize].c; + if(stack[ssize].d == 0) { + tr_copy(ISA, SA, first, a, b, last, ISAd - ISA); + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA); + } + STACK_POP5(ISAd, first, last, limit, trlink); + } else { + /* sorted partition */ + if(0 <= *first) { + a = first; + do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a)); + first = a; + } + if(first < last) { + a = first; do { *a = ~*a; } while(*++a < 0); + next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1; + if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } } + + /* push */ + if(trbudget_check(budget, a - first)) { + if((a - first) <= (last - a)) { + STACK_PUSH5(ISAd, a, last, -3, trlink); + ISAd += incr, last = a, limit = next; + } else { + if(1 < (last - a)) { + STACK_PUSH5(ISAd + incr, first, a, next, trlink); + first = a, limit = -3; + } else { + ISAd += incr, last = a, limit = next; + } + } + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + if(1 < (last - a)) { + first = a, limit = -3; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + continue; + } + + if((last - first) <= TR_INSERTIONSORT_THRESHOLD) { + tr_insertionsort(ISAd, first, last); + limit = -3; + continue; + } + + if(limit-- == 0) { + tr_heapsort(ISAd, first, last - first); + for(a = last - 1; first < a; a = b) { + for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; } + } + limit = -3; + continue; + } + + /* choose pivot */ + a = tr_pivot(ISAd, first, last); + SWAP(*first, *a); + v = ISAd[*first]; + + /* partition */ + tr_partition(ISAd, first, first + 1, last, &a, &b, v); + if((last - first) != (b - a)) { + next = (ISA[*a] != v) ? tr_ilg(b - a) : -1; + + /* update ranks */ + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } } + + /* push */ + if((1 < (b - a)) && (trbudget_check(budget, b - a))) { + if((a - first) <= (last - b)) { + if((last - b) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((a - first) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + if((a - first) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((last - b) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } + } else { + if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + first = b; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + last = a; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } + } else { + if(trbudget_check(budget, last - first)) { + limit = tr_ilg(last - first), ISAd += incr; + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } +#undef STACK_SIZE +} + + + +/*---------------------------------------------------------------------------*/ + +/* Tandem repeat sort */ +static +void +trsort(int *ISA, int *SA, int n, int depth) { + int *ISAd; + int *first, *last; + trbudget_t budget; + int t, skip, unsorted; + + trbudget_init(&budget, tr_ilg(n) * 2 / 3, n); +/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */ + for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) { + first = SA; + skip = 0; + unsorted = 0; + do { + if((t = *first) < 0) { first -= t; skip += t; } + else { + if(skip != 0) { *(first + skip) = skip; skip = 0; } + last = SA + ISA[t] + 1; + if(1 < (last - first)) { + budget.count = 0; + tr_introsort(ISA, ISAd, SA, first, last, &budget); + if(budget.count != 0) { unsorted += budget.count; } + else { skip = first - last; } + } else if((last - first) == 1) { + skip = -1; + } + first = last; + } + } while(first < (SA + n)); + if(skip != 0) { *(first + skip) = skip; } + if(unsorted == 0) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Sorts suffixes of type B*. */ +static +int +sort_typeBstar(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int openMP) { + int *PAb, *ISAb, *buf; +#ifdef LIBBSC_OPENMP + int *curbuf; + int l; +#endif + int i, j, k, t, m, bufsize; + int c0, c1; +#ifdef LIBBSC_OPENMP + int d0, d1; +#endif + (void)openMP; + + /* Initialize bucket arrays. */ + for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; } + for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; } + + /* Count the number of occurrences of the first one or two characters of each + type A, B and B* suffix. Moreover, store the beginning position of all + type B* suffixes into the array SA. */ + for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) { + /* type A suffix. */ + do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1)); + if(0 <= i) { + /* type B* suffix. */ + ++BUCKET_BSTAR(c0, c1); + SA[--m] = i; + /* type B suffix. */ + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { + ++BUCKET_B(c0, c1); + } + } + } + m = n - m; +/* +note: + A type B* suffix is lexicographically smaller than a type B suffix that + begins with the same first two characters. +*/ + + /* Calculate the index of start/end point of each bucket. */ + for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) { + t = i + BUCKET_A(c0); + BUCKET_A(c0) = i + j; /* start point */ + i = t + BUCKET_B(c0, c0); + for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) { + j += BUCKET_BSTAR(c0, c1); + BUCKET_BSTAR(c0, c1) = j; /* end point */ + i += BUCKET_B(c0, c1); + } + } + + if(0 < m) { + /* Sort the type B* suffixes by their first two characters. */ + PAb = SA + n - m; ISAb = SA + m; + for(i = m - 2; 0 <= i; --i) { + t = PAb[i], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = i; + } + t = PAb[m - 1], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = m - 1; + + /* Sort the type B* substrings using sssort. */ +#ifdef LIBBSC_OPENMP + if (openMP) + { + buf = SA + m; + c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m; +#pragma omp parallel default(shared) private(bufsize, curbuf, k, l, d0, d1) + { + bufsize = (n - (2 * m)) / omp_get_num_threads(); + curbuf = buf + omp_get_thread_num() * bufsize; + k = 0; + for(;;) { + #pragma omp critical(sssort_lock) + { + if(0 < (l = j)) { + d0 = c0, d1 = c1; + do { + k = BUCKET_BSTAR(d0, d1); + if(--d1 <= d0) { + d1 = ALPHABET_SIZE - 1; + if(--d0 < 0) { break; } + } + } while(((l - k) <= 1) && (0 < (l = k))); + c0 = d0, c1 = d1, j = k; + } + } + if(l == 0) { break; } + sssort(T, PAb, SA + k, SA + l, + curbuf, bufsize, 2, n, *(SA + k) == (m - 1)); + } + } + } + else + { + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } + } +#else + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } +#endif + + /* Compute ranks of type B* substrings. */ + for(i = m - 1; 0 <= i; --i) { + if(0 <= SA[i]) { + j = i; + do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i])); + SA[i + 1] = i - j; + if(i <= 0) { break; } + } + j = i; + do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0); + ISAb[SA[i]] = j; + } + + /* Construct the inverse suffix array of type B* suffixes using trsort. */ + trsort(ISAb, SA, m, 1); + + /* Set the sorted order of tyoe B* suffixes. */ + for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) { + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { } + if(0 <= i) { + t = i; + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { } + SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t; + } + } + + /* Calculate the index of start/end point of each bucket. */ + BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */ + for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) { + i = BUCKET_A(c0 + 1) - 1; + for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) { + t = i - BUCKET_B(c0, c1); + BUCKET_B(c0, c1) = i; /* end point */ + + /* Move all type B* suffixes to the correct position. */ + for(i = t, j = BUCKET_BSTAR(c0, c1); + j <= k; + --i, --k) { SA[i] = SA[k]; } + } + BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */ + BUCKET_B(c0, c0) = i; /* end point */ + } + } + + return m; +} + +/* Constructs the suffix array by using the sorted order of type B* suffixes. */ +static +void +construct_SA(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + *j = ~s; + c0 = T[--s]; + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else { + assert(((s == 0) && (T[s] == c1)) || (s < 0)); + *j = ~s; + } + } + } + } + + /* Construct the suffix array by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + if((s == 0) || (T[s - 1] < c0)) { s = ~s; } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else { + assert(s < 0); + *i = ~s; + } + } +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~((int)T[n - 2]) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + *i = c0; + if((0 < s) && (T[s - 1] < c0)) { s = ~((int)T[s - 1]); } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +int +construct_BWT_indexes(const unsigned char *T, int *SA, + int *bucket_A, int *bucket_B, + int n, int m, + unsigned char * num_indexes, int * indexes) { + int *i, *j, *k, *orig; + int s; + int c0, c1, c2; + + int mod = n / 8; + { + mod |= mod >> 1; mod |= mod >> 2; + mod |= mod >> 4; mod |= mod >> 8; + mod |= mod >> 16; mod >>= 1; + + *num_indexes = (unsigned char)((n - 1) / (mod + 1)); + } + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = j - SA; + + c0 = T[--s]; + *j = ~((int)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + if (T[n - 2] < c2) { + if (((n - 1) & mod) == 0) indexes[(n - 1) / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[n - 2]); + } + else { + *k++ = n - 1; + } + + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = i - SA; + + c0 = T[--s]; + *i = c0; + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + if((0 < s) && (T[s - 1] < c0)) { + if ((s & mod) == 0) indexes[s / (mod + 1) - 1] = k - SA; + *k++ = ~((int)T[s - 1]); + } else + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP) { + int *bucket_A, *bucket_B; + int m; + int err = 0; + + /* Check arguments. */ + if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; } + else if(n == 0) { return 0; } + else if(n == 1) { SA[0] = 0; return 0; } + else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; } + + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Suffixsort. */ + if((bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, SA, bucket_A, bucket_B, n, openMP); + construct_SA(T, SA, bucket_A, bucket_B, n, m); + } else { + err = -2; + } + + free(bucket_B); + free(bucket_A); + + return err; +} + +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP) { + int *B; + int *bucket_A, *bucket_B; + int m, pidx, i; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0)) { return -1; } + else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; } + + if((B = A) == NULL) { B = (int *)malloc((size_t)(n + 1) * sizeof(int)); } + bucket_A = (int *)malloc(BUCKET_A_SIZE * sizeof(int)); + bucket_B = (int *)malloc(BUCKET_B_SIZE * sizeof(int)); + + /* Burrows-Wheeler Transform. */ + if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, B, bucket_A, bucket_B, n, openMP); + + if (num_indexes == NULL || indexes == NULL) { + pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m); + } else { + pidx = construct_BWT_indexes(T, B, bucket_A, bucket_B, n, m, num_indexes, indexes); + } + + /* Copy to output string. */ + U[0] = T[n - 1]; + for(i = 0; i < pidx; ++i) { U[i + 1] = (unsigned char)B[i]; } + for(i += 1; i < n; ++i) { U[i] = (unsigned char)B[i]; } + pidx += 1; + } else { + pidx = -2; + } + + free(bucket_B); + free(bucket_A); + if(A == NULL) { free(B); } + + return pidx; +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/divsufsort.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,67 @@ +/* + * divsufsort.h for libdivsufsort-lite + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_H +#define _DIVSUFSORT_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + + +/*- Prototypes -*/ + +/** + * Constructs the suffix array of a given string. + * @param T [0..n-1] The input string. + * @param SA [0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @param openMP enables OpenMP optimization. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +int +divsufsort(const unsigned char *T, int *SA, int n, int openMP); + +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T [0..n-1] The input string. + * @param U [0..n-1] The output string. (can be T) + * @param A [0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param num_indexes The length of secondary indexes array. (can be NULL) + * @param indexes The secondary indexes array. (can be NULL) + * @param openMP enables OpenMP optimization. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +int +divbwt(const unsigned char *T, unsigned char *U, int *A, int n, unsigned char * num_indexes, int * indexes, int openMP); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_H */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,1012 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + + +/*-************************************** +* Tuning parameters +****************************************/ +#define ZDICT_MAX_SAMPLES_SIZE (2000U << 20) +#define ZDICT_MIN_SAMPLES_SIZE 512 + + +/*-************************************** +* Compiler Options +****************************************/ +/* Unix Large Files support (>4GB) */ +#define _FILE_OFFSET_BITS 64 +#if (defined(__sun__) && (!defined(__LP64__))) /* Sun Solaris 32-bits requires specific definitions */ +# define _LARGEFILE_SOURCE +#elif ! defined(__LP64__) /* No point defining Large file for 64 bit */ +# define _LARGEFILE64_SOURCE +#endif + + +/*-************************************* +* Dependencies +***************************************/ +#include /* malloc, free */ +#include /* memset */ +#include /* fprintf, fopen, ftello64 */ +#include /* clock */ + +#include "mem.h" /* read */ +#include "error_private.h" +#include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */ +#define HUF_STATIC_LINKING_ONLY +#include "huf.h" +#include "zstd_internal.h" /* includes zstd.h */ +#include "xxhash.h" +#include "divsufsort.h" +#ifndef ZDICT_STATIC_LINKING_ONLY +# define ZDICT_STATIC_LINKING_ONLY +#endif +#include "zdict.h" + + +/*-************************************* +* Constants +***************************************/ +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +#define DICTLISTSIZE_DEFAULT 10000 + +#define NOISELENGTH 32 + +#define MINRATIO 4 +static const int g_compressionLevel_default = 5; +static const U32 g_selectivity_default = 9; +static const size_t g_provision_entropySize = 200; +static const size_t g_min_fast_dictContent = 192; + + +/*-************************************* +* Console display +***************************************/ +#define DISPLAY(...) { fprintf(stderr, __VA_ARGS__); fflush( stderr ); } +#define DISPLAYLEVEL(l, ...) if (notificationLevel>=l) { DISPLAY(__VA_ARGS__); } /* 0 : no display; 1: errors; 2: default; 3: details; 4: debug */ + +static clock_t ZDICT_clockSpan(clock_t nPrevious) { return clock() - nPrevious; } + +static void ZDICT_printHex(const void* ptr, size_t length) +{ + const BYTE* const b = (const BYTE*)ptr; + size_t u; + for (u=0; u126) c = '.'; /* non-printable char */ + DISPLAY("%c", c); + } +} + + +/*-******************************************************** +* Helper functions +**********************************************************/ +unsigned ZDICT_isError(size_t errorCode) { return ERR_isError(errorCode); } + +const char* ZDICT_getErrorName(size_t errorCode) { return ERR_getErrorName(errorCode); } + +unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize) +{ + if (dictSize < 8) return 0; + if (MEM_readLE32(dictBuffer) != ZSTD_DICT_MAGIC) return 0; + return MEM_readLE32((const char*)dictBuffer + 4); +} + + +/*-******************************************************** +* Dictionary training functions +**********************************************************/ +static unsigned ZDICT_NbCommonBytes (register size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanForward64( &r, (U64)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctzll((U64)val) >> 3); +# else + static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; + return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r=0; + _BitScanForward( &r, (U32)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_ctz((U32)val) >> 3); +# else + static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; + return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; +# endif + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { +# if defined(_MSC_VER) && defined(_WIN64) + unsigned long r = 0; + _BitScanReverse64( &r, val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clzll(val) >> 3); +# else + unsigned r; + const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ + if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } + if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } + r += (!val); + return r; +# endif + } else { /* 32 bits */ +# if defined(_MSC_VER) + unsigned long r = 0; + _BitScanReverse( &r, (unsigned long)val ); + return (unsigned)(r>>3); +# elif defined(__GNUC__) && (__GNUC__ >= 3) + return (__builtin_clz((U32)val) >> 3); +# else + unsigned r; + if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } + r += (!val); + return r; +# endif + } } +} + + +/*! ZDICT_count() : + Count the nb of common bytes between 2 pointers. + Note : this function presumes end of buffer followed by noisy guard band. +*/ +static size_t ZDICT_count(const void* pIn, const void* pMatch) +{ + const char* const pStart = (const char*)pIn; + for (;;) { + size_t const diff = MEM_readST(pMatch) ^ MEM_readST(pIn); + if (!diff) { + pIn = (const char*)pIn+sizeof(size_t); + pMatch = (const char*)pMatch+sizeof(size_t); + continue; + } + pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + return (size_t)((const char*)pIn - pStart); + } +} + + +typedef struct { + U32 pos; + U32 length; + U32 savings; +} dictItem; + +static void ZDICT_initDictItem(dictItem* d) +{ + d->pos = 1; + d->length = 0; + d->savings = (U32)(-1); +} + + +#define LLIMIT 64 /* heuristic determined experimentally */ +#define MINMATCHLENGTH 7 /* heuristic determined experimentally */ +static dictItem ZDICT_analyzePos( + BYTE* doneMarks, + const int* suffix, U32 start, + const void* buffer, U32 minRatio, U32 notificationLevel) +{ + U32 lengthList[LLIMIT] = {0}; + U32 cumulLength[LLIMIT] = {0}; + U32 savings[LLIMIT] = {0}; + const BYTE* b = (const BYTE*)buffer; + size_t length; + size_t maxLength = LLIMIT; + size_t pos = suffix[start]; + U32 end = start; + dictItem solution; + + /* init */ + memset(&solution, 0, sizeof(solution)); + doneMarks[pos] = 1; + + /* trivial repetition cases */ + if ( (MEM_read16(b+pos+0) == MEM_read16(b+pos+2)) + ||(MEM_read16(b+pos+1) == MEM_read16(b+pos+3)) + ||(MEM_read16(b+pos+2) == MEM_read16(b+pos+4)) ) { + /* skip and mark segment */ + U16 u16 = MEM_read16(b+pos+4); + U32 u, e = 6; + while (MEM_read16(b+pos+e) == u16) e+=2 ; + if (b[pos+e] == b[pos+e-1]) e++; + for (u=1; u=MINMATCHLENGTH); + + /* look backward */ + do { + length = ZDICT_count(b + pos, b + *(suffix+start-1)); + if (length >=MINMATCHLENGTH) start--; + } while(length >= MINMATCHLENGTH); + + /* exit if not found a minimum nb of repetitions */ + if (end-start < minRatio) { + U32 idx; + for(idx=start; idx= %i at pos %7u ", (U32)(end-start), MINMATCHLENGTH, (U32)pos); + DISPLAYLEVEL(4, "\n"); + + for (searchLength = MINMATCHLENGTH ; ; searchLength++) { + BYTE currentChar = 0; + U32 currentCount = 0; + U32 currentID = refinedStart; + U32 id; + U32 selectedCount = 0; + U32 selectedID = currentID; + for (id =refinedStart; id < refinedEnd; id++) { + if (b[ suffix[id] + searchLength] != currentChar) { + if (currentCount > selectedCount) { + selectedCount = currentCount; + selectedID = currentID; + } + currentID = id; + currentChar = b[ suffix[id] + searchLength]; + currentCount = 0; + } + currentCount ++; + } + if (currentCount > selectedCount) { /* for last */ + selectedCount = currentCount; + selectedID = currentID; + } + + if (selectedCount < minRatio) + break; + refinedStart = selectedID; + refinedEnd = refinedStart + selectedCount; + } + + /* evaluate gain based on new ref */ + start = refinedStart; + pos = suffix[refinedStart]; + end = start; + memset(lengthList, 0, sizeof(lengthList)); + + /* look forward */ + do { + end++; + length = ZDICT_count(b + pos, b + suffix[end]); + if (length >= LLIMIT) length = LLIMIT-1; + lengthList[length]++; + } while (length >=MINMATCHLENGTH); + + /* look backward */ + length = MINMATCHLENGTH; + while ((length >= MINMATCHLENGTH) & (start > 0)) { + length = ZDICT_count(b + pos, b + suffix[start - 1]); + if (length >= LLIMIT) length = LLIMIT - 1; + lengthList[length]++; + if (length >= MINMATCHLENGTH) start--; + } + + /* largest useful length */ + memset(cumulLength, 0, sizeof(cumulLength)); + cumulLength[maxLength-1] = lengthList[maxLength-1]; + for (i=(int)(maxLength-2); i>=0; i--) + cumulLength[i] = cumulLength[i+1] + lengthList[i]; + + for (i=LLIMIT-1; i>=MINMATCHLENGTH; i--) if (cumulLength[i]>=minRatio) break; + maxLength = i; + + /* reduce maxLength in case of final into repetitive data */ + { U32 l = (U32)maxLength; + BYTE const c = b[pos + maxLength-1]; + while (b[pos+l-2]==c) l--; + maxLength = l; + } + if (maxLength < MINMATCHLENGTH) return solution; /* skip : no long-enough solution */ + + /* calculate savings */ + savings[5] = 0; + for (i=MINMATCHLENGTH; i<=(int)maxLength; i++) + savings[i] = savings[i-1] + (lengthList[i] * (i-3)); + + DISPLAYLEVEL(4, "Selected ref at position %u, of length %u : saves %u (ratio: %.2f) \n", + (U32)pos, (U32)maxLength, savings[maxLength], (double)savings[maxLength] / maxLength); + + solution.pos = (U32)pos; + solution.length = (U32)maxLength; + solution.savings = savings[maxLength]; + + /* mark positions done */ + { U32 id; + for (id=start; id solution.length) length = solution.length; + } + pEnd = (U32)(testedPos + length); + for (p=testedPos; ppos; + const U32 eltEnd = elt.pos + elt.length; + + /* tail overlap */ + U32 u; for (u=1; u elt.pos) && (table[u].pos <= eltEnd)) { /* overlap, existing > new */ + /* append */ + U32 addedLength = table[u].pos - elt.pos; + table[u].length += addedLength; + table[u].pos = elt.pos; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + table[u].savings += elt.length / 8; /* rough approx bonus */ + elt = table[u]; + /* sort : improve rank */ + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } } + + /* front overlap */ + for (u=1; u= elt.pos) && (table[u].pos < elt.pos)) { /* overlap, existing < new */ + /* append */ + int addedLength = (int)eltEnd - (table[u].pos + table[u].length); + table[u].savings += elt.length / 8; /* rough approx bonus */ + if (addedLength > 0) { /* otherwise, elt fully included into existing */ + table[u].length += addedLength; + table[u].savings += elt.savings * addedLength / elt.length; /* rough approx */ + } + /* sort : improve rank */ + elt = table[u]; + while ((u>1) && (table[u-1].savings < elt.savings)) + table[u] = table[u-1], u--; + table[u] = elt; + return u; + } } + + return 0; +} + + +static void ZDICT_removeDictItem(dictItem* table, U32 id) +{ + /* convention : first element is nb of elts */ + U32 const max = table->pos; + U32 u; + if (!id) return; /* protection, should never happen */ + for (u=id; upos--; +} + + +static void ZDICT_insertDictItem(dictItem* table, U32 maxSize, dictItem elt) +{ + /* merge if possible */ + U32 mergeId = ZDICT_checkMerge(table, elt, 0); + if (mergeId) { + U32 newMerge = 1; + while (newMerge) { + newMerge = ZDICT_checkMerge(table, table[mergeId], mergeId); + if (newMerge) ZDICT_removeDictItem(table, mergeId); + mergeId = newMerge; + } + return; + } + + /* insert */ + { U32 current; + U32 nextElt = table->pos; + if (nextElt >= maxSize) nextElt = maxSize-1; + current = nextElt-1; + while (table[current].savings < elt.savings) { + table[current+1] = table[current]; + current--; + } + table[current+1] = elt; + table->pos = nextElt+1; + } +} + + +static U32 ZDICT_dictSize(const dictItem* dictList) +{ + U32 u, dictSize = 0; + for (u=1; u=l) { \ + if (ZDICT_clockSpan(displayClock) > refreshRate) \ + { displayClock = clock(); DISPLAY(__VA_ARGS__); \ + if (notificationLevel>=4) fflush(stdout); } } + + /* init */ + DISPLAYLEVEL(2, "\r%70s\r", ""); /* clean display line */ + if (!suffix0 || !reverseSuffix || !doneMarks || !filePos) { + result = ERROR(memory_allocation); + goto _cleanup; + } + if (minRatio < MINRATIO) minRatio = MINRATIO; + memset(doneMarks, 0, bufferSize+16); + + /* limit sample set size (divsufsort limitation)*/ + if (bufferSize > ZDICT_MAX_SAMPLES_SIZE) DISPLAYLEVEL(3, "sample set too large : reduced to %u MB ...\n", (U32)(ZDICT_MAX_SAMPLES_SIZE>>20)); + while (bufferSize > ZDICT_MAX_SAMPLES_SIZE) bufferSize -= fileSizes[--nbFiles]; + + /* sort */ + DISPLAYLEVEL(2, "sorting %u files of total size %u MB ...\n", nbFiles, (U32)(bufferSize>>20)); + { int const divSuftSortResult = divsufsort((const unsigned char*)buffer, suffix, (int)bufferSize, 0); + if (divSuftSortResult != 0) { result = ERROR(GENERIC); goto _cleanup; } + } + suffix[bufferSize] = (int)bufferSize; /* leads into noise */ + suffix0[0] = (int)bufferSize; /* leads into noise */ + /* build reverse suffix sort */ + { size_t pos; + for (pos=0; pos < bufferSize; pos++) + reverseSuffix[suffix[pos]] = (U32)pos; + /* note filePos tracks borders between samples. + It's not used at this stage, but planned to become useful in a later update */ + filePos[0] = 0; + for (pos=1; pos> 21); + } +} + + +typedef struct +{ + ZSTD_CCtx* ref; + ZSTD_CCtx* zc; + void* workPlace; /* must be ZSTD_BLOCKSIZE_ABSOLUTEMAX allocated */ +} EStats_ress_t; + +#define MAXREPOFFSET 1024 + +static void ZDICT_countEStats(EStats_ress_t esr, ZSTD_parameters params, + U32* countLit, U32* offsetcodeCount, U32* matchlengthCount, U32* litlengthCount, U32* repOffsets, + const void* src, size_t srcSize, U32 notificationLevel) +{ + size_t const blockSizeMax = MIN (ZSTD_BLOCKSIZE_ABSOLUTEMAX, 1 << params.cParams.windowLog); + size_t cSize; + + if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ + { size_t const errorCode = ZSTD_copyCCtx(esr.zc, esr.ref, 0); + if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_copyCCtx failed \n"); return; } + } + cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_ABSOLUTEMAX, src, srcSize); + if (ZSTD_isError(cSize)) { DISPLAYLEVEL(1, "warning : could not compress sample size %u \n", (U32)srcSize); return; } + + if (cSize) { /* if == 0; block is not compressible */ + const seqStore_t* seqStorePtr = ZSTD_getSeqStore(esr.zc); + + /* literals stats */ + { const BYTE* bytePtr; + for(bytePtr = seqStorePtr->litStart; bytePtr < seqStorePtr->lit; bytePtr++) + countLit[*bytePtr]++; + } + + /* seqStats */ + { U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + ZSTD_seqToCodes(seqStorePtr); + + { const BYTE* codePtr = seqStorePtr->ofCode; + U32 u; + for (u=0; umlCode; + U32 u; + for (u=0; ullCode; + U32 u; + for (u=0; u= 2) { /* rep offsets */ + const seqDef* const seq = seqStorePtr->sequencesStart; + U32 offset1 = seq[0].offset - 3; + U32 offset2 = seq[1].offset - 3; + if (offset1 >= MAXREPOFFSET) offset1 = 0; + if (offset2 >= MAXREPOFFSET) offset2 = 0; + repOffsets[offset1] += 3; + repOffsets[offset2] += 1; + } } } +} + +/* +static size_t ZDICT_maxSampleSize(const size_t* fileSizes, unsigned nbFiles) +{ + unsigned u; + size_t max=0; + for (u=0; u0; u--) { + offsetCount_t tmp; + if (table[u-1].count >= table[u].count) break; + tmp = table[u-1]; + table[u-1] = table[u]; + table[u] = tmp; + } +} + + +#define OFFCODE_MAX 30 /* only applicable to first block */ +static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, + unsigned compressionLevel, + const void* srcBuffer, const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize, + unsigned notificationLevel) +{ + U32 countLit[256]; + HUF_CREATE_STATIC_CTABLE(hufTable, 255); + U32 offcodeCount[OFFCODE_MAX+1]; + short offcodeNCount[OFFCODE_MAX+1]; + U32 offcodeMax = ZSTD_highbit32((U32)(dictBufferSize + 128 KB)); + U32 matchLengthCount[MaxML+1]; + short matchLengthNCount[MaxML+1]; + U32 litLengthCount[MaxLL+1]; + short litLengthNCount[MaxLL+1]; + U32 repOffset[MAXREPOFFSET]; + offsetCount_t bestRepOffset[ZSTD_REP_NUM+1]; + EStats_ress_t esr; + ZSTD_parameters params; + U32 u, huffLog = 11, Offlog = OffFSELog, mlLog = MLFSELog, llLog = LLFSELog, total; + size_t pos = 0, errorCode; + size_t eSize = 0; + size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); + size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); + BYTE* dstPtr = (BYTE*)dstBuffer; + + /* init */ + esr.ref = ZSTD_createCCtx(); + esr.zc = ZSTD_createCCtx(); + esr.workPlace = malloc(ZSTD_BLOCKSIZE_ABSOLUTEMAX); + if (!esr.ref || !esr.zc || !esr.workPlace) { + eSize = ERROR(memory_allocation); + DISPLAYLEVEL(1, "Not enough memory \n"); + goto _cleanup; + } + if (offcodeMax>OFFCODE_MAX) { eSize = ERROR(dictionary_wrong); goto _cleanup; } /* too large dictionary */ + for (u=0; u<256; u++) countLit[u]=1; /* any character must be described */ + for (u=0; u<=offcodeMax; u++) offcodeCount[u]=1; + for (u=0; u<=MaxML; u++) matchLengthCount[u]=1; + for (u=0; u<=MaxLL; u++) litLengthCount[u]=1; + memset(repOffset, 0, sizeof(repOffset)); + repOffset[1] = repOffset[4] = repOffset[8] = 1; + memset(bestRepOffset, 0, sizeof(bestRepOffset)); + if (compressionLevel==0) compressionLevel=g_compressionLevel_default; + params = ZSTD_getParams(compressionLevel, averageSampleSize, dictBufferSize); + { size_t const beginResult = ZSTD_compressBegin_advanced(esr.ref, dictBuffer, dictBufferSize, params, 0); + if (ZSTD_isError(beginResult)) { + eSize = ERROR(GENERIC); + DISPLAYLEVEL(1, "error : ZSTD_compressBegin_advanced failed \n"); + goto _cleanup; + } } + + /* collect stats on all files */ + for (u=0; u 30) ? MINRATIO : nbSamples >> selectivity; + size_t const targetDictSize = maxDictSize; + size_t const samplesBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + size_t dictSize = 0; + U32 const notificationLevel = params.notificationLevel; + + /* checks */ + if (!dictList) return ERROR(memory_allocation); + if (maxDictSize <= g_provision_entropySize + g_min_fast_dictContent) { free(dictList); return ERROR(dstSize_tooSmall); } + if (samplesBuffSize < ZDICT_MIN_SAMPLES_SIZE) { free(dictList); return 0; } /* not enough source to create dictionary */ + + /* init */ + ZDICT_initDictItem(dictList); + + /* build dictionary */ + ZDICT_trainBuffer(dictList, dictListSize, + samplesBuffer, samplesBuffSize, + samplesSizes, nbSamples, + minRep, notificationLevel); + + /* display best matches */ + if (params.notificationLevel>= 3) { + U32 const nb = MIN(25, dictList[0].pos); + U32 const dictContentSize = ZDICT_dictSize(dictList); + U32 u; + DISPLAYLEVEL(3, "\n %u segments found, of total size %u \n", dictList[0].pos-1, dictContentSize); + DISPLAYLEVEL(3, "list %u best segments \n", nb-1); + for (u=1; u samplesBuffSize) || ((pos + length) > samplesBuffSize)) + return ERROR(GENERIC); /* should never happen */ + DISPLAYLEVEL(3, "%3u:%3u bytes at pos %8u, savings %7u bytes |", + u, length, pos, dictList[u].savings); + ZDICT_printHex((const char*)samplesBuffer+pos, printedLength); + DISPLAYLEVEL(3, "| \n"); + } } + + + /* create dictionary */ + { U32 dictContentSize = ZDICT_dictSize(dictList); + if (dictContentSize < targetDictSize/3) { + DISPLAYLEVEL(2, "! warning : selected content significantly smaller than requested (%u < %u) \n", dictContentSize, (U32)maxDictSize); + if (minRep > MINRATIO) { + DISPLAYLEVEL(2, "! consider increasing selectivity to produce larger dictionary (-s%u) \n", selectivity+1); + DISPLAYLEVEL(2, "! note : larger dictionaries are not necessarily better, test its efficiency on samples \n"); + } + if (samplesBuffSize < 10 * targetDictSize) + DISPLAYLEVEL(2, "! consider increasing the number of samples (total size : %u MB)\n", (U32)(samplesBuffSize>>20)); + } + + if ((dictContentSize > targetDictSize*3) && (nbSamples > 2*MINRATIO) && (selectivity>1)) { + U32 proposedSelectivity = selectivity-1; + while ((nbSamples >> proposedSelectivity) <= MINRATIO) { proposedSelectivity--; } + DISPLAYLEVEL(2, "! note : calculated dictionary significantly larger than requested (%u > %u) \n", dictContentSize, (U32)maxDictSize); + DISPLAYLEVEL(2, "! consider increasing dictionary size, or produce denser dictionary (-s%u) \n", proposedSelectivity); + DISPLAYLEVEL(2, "! always test dictionary efficiency on samples \n"); + } + + /* limit dictionary size */ + { U32 const max = dictList->pos; /* convention : nb of useful elts within dictList */ + U32 currentSize = 0; + U32 n; for (n=1; n targetDictSize) { currentSize -= dictList[n].length; break; } + } + dictList->pos = n; + dictContentSize = currentSize; + } + + /* build dict content */ + { U32 u; + BYTE* ptr = (BYTE*)dictBuffer + maxDictSize; + for (u=1; upos; u++) { + U32 l = dictList[u].length; + ptr -= l; + if (ptr<(BYTE*)dictBuffer) { free(dictList); return ERROR(GENERIC); } /* should not happen */ + memcpy(ptr, (const char*)samplesBuffer+dictList[u].pos, l); + } } + + dictSize = ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, maxDictSize, + samplesBuffer, samplesSizes, nbSamples, + params); + } + + /* clean up */ + free(dictList); + return dictSize; +} + + +/* issue : samplesBuffer need to be followed by a noisy guard band. +* work around : duplicate the buffer, and add the noise */ +size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t params) +{ + size_t result; + void* newBuff; + size_t const sBuffSize = ZDICT_totalSampleSize(samplesSizes, nbSamples); + if (sBuffSize < ZDICT_MIN_SAMPLES_SIZE) return 0; /* not enough content => no dictionary */ + + newBuff = malloc(sBuffSize + NOISELENGTH); + if (!newBuff) return ERROR(memory_allocation); + + memcpy(newBuff, samplesBuffer, sBuffSize); + ZDICT_fillNoise((char*)newBuff + sBuffSize, NOISELENGTH); /* guard band, for end of buffer condition */ + + result = ZDICT_trainFromBuffer_unsafe( + dictBuffer, dictBufferCapacity, + newBuff, samplesSizes, nbSamples, + params); + free(newBuff); + return result; +} + + +size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_trainFromBuffer_advanced(dictBuffer, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +} + +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples) +{ + ZDICT_params_t params; + memset(¶ms, 0, sizeof(params)); + return ZDICT_addEntropyTablesFromBuffer_advanced(dictBuffer, dictContentSize, dictBufferCapacity, + samplesBuffer, samplesSizes, nbSamples, + params); +} diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/dictBuilder/zdict.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,111 @@ +/** + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#ifndef DICTBUILDER_H_001 +#define DICTBUILDER_H_001 + +#if defined (__cplusplus) +extern "C" { +#endif + + +/*====== Dependencies ======*/ +#include /* size_t */ + + +/*====== Export for Windows ======*/ +/*! +* ZSTD_DLL_EXPORT : +* Enable exporting of functions when building a Windows DLL +*/ +#if defined(_WIN32) && defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_API __declspec(dllexport) +#else +# define ZDICTLIB_API +#endif + + +/*! ZDICT_trainFromBuffer() : + Train a dictionary from an array of samples. + Samples must be stored concatenated in a single flat buffer `samplesBuffer`, + supplied with an array of sizes `samplesSizes`, providing the size of each sample, in order. + The resulting dictionary will be saved into `dictBuffer`. + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`) + or an error code, which can be tested with ZDICT_isError(). + Tips : In general, a reasonable dictionary has a size of ~ 100 KB. + It's obviously possible to target smaller or larger ones, just by specifying different `dictBufferCapacity`. + In general, it's recommended to provide a few thousands samples, but this can vary a lot. + It's recommended that total size of all samples be about ~x100 times the target size of dictionary. +*/ +ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + +/*====== Helper functions ======*/ +ZDICTLIB_API unsigned ZDICT_getDictID(const void* dictBuffer, size_t dictSize); /**< extracts dictID; @return zero if error (not a valid dictionary) */ +ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); +ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); + + + +#ifdef ZDICT_STATIC_LINKING_ONLY + +/* ==================================================================================== + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ==================================================================================== */ + +typedef struct { + unsigned selectivityLevel; /* 0 means default; larger => select more => larger dictionary */ + int compressionLevel; /* 0 means default; target a specific zstd compression level */ + unsigned notificationLevel; /* Write to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /* 0 means auto mode (32-bits random value); other : force dictID value */ + unsigned reserved[2]; /* reserved space for future parameters */ +} ZDICT_params_t; + + +/*! ZDICT_trainFromBuffer_advanced() : + Same as ZDICT_trainFromBuffer() with control over more parameters. + `parameters` is optional and can be provided with values set to 0 to mean "default". + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferSize`), + or an error code, which can be tested by ZDICT_isError(). + note : ZDICT_trainFromBuffer_advanced() will send notifications into stderr if instructed to, using notificationLevel>0. +*/ +size_t ZDICT_trainFromBuffer_advanced(void* dictBuffer, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, + ZDICT_params_t parameters); + + +/*! ZDICT_addEntropyTablesFromBuffer() : + + Given a content-only dictionary (built using any 3rd party algorithm), + add entropy tables computed from an array of samples. + Samples must be stored concatenated in a flat buffer `samplesBuffer`, + supplied with an array of sizes `samplesSizes`, providing the size of each sample in order. + + The input dictionary content must be stored *at the end* of `dictBuffer`. + Its size is `dictContentSize`. + The resulting dictionary with added entropy tables will be *written back to `dictBuffer`*, + starting from its beginning. + @return : size of dictionary stored into `dictBuffer` (<= `dictBufferCapacity`). +*/ +size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, + const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); + + + +#endif /* ZDICT_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif + +#endif /* DICTBUILDER_H_001 */ diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/LICENSE mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/LICENSE --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/LICENSE 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/LICENSE 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) 2016-present, Facebook, Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook nor the names of its contributors may be used to + endorse or promote products derived from this software without specific + prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/PATENTS mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/PATENTS --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/PATENTS 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/PATENTS 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,33 @@ +Additional Grant of Patent Rights Version 2 + +"Software" means the Zstandard software distributed by Facebook, Inc. + +Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software +("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable +(subject to the termination provision below) license under any Necessary +Claims, to make, have made, use, sell, offer to sell, import, and otherwise +transfer the Software. For avoidance of doubt, no license is granted under +Facebook’s rights in any patent claims that are infringed by (i) modifications +to the Software made by you or any third party or (ii) the Software in +combination with any software or other technology. + +The license granted hereunder will terminate, automatically and without notice, +if you (or any of your subsidiaries, corporate affiliates or agents) initiate +directly or indirectly, or take a direct financial interest in, any Patent +Assertion: (i) against Facebook or any of its subsidiaries or corporate +affiliates, (ii) against any party if such Patent Assertion arises in whole or +in part from any software, technology, product or service of Facebook or any of +its subsidiaries or corporate affiliates, or (iii) against any party relating +to the Software. Notwithstanding the foregoing, if Facebook or any of its +subsidiaries or corporate affiliates files a lawsuit alleging patent +infringement against you in the first instance, and you respond by filing a +patent infringement counterclaim in that lawsuit against that party that is +unrelated to the Software, the license granted hereunder will not terminate +under section (i) of this paragraph due to such counterclaim. + +A "Necessary Claim" is a claim of a patent owned by Facebook that is +necessarily infringed by the Software standing alone. + +A "Patent Assertion" is any lawsuit or other action alleging direct, indirect, +or contributory infringement or inducement to infringe any patent, including a +cross-claim or counterclaim. diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/zstd.h mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/zstd.h --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd/zstd.h 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd/zstd.h 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,673 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under the BSD-style license found in the + * LICENSE file in the root directory of this source tree. An additional grant + * of patent rights can be found in the PATENTS file in the same directory. + */ + +#if defined (__cplusplus) +extern "C" { +#endif + +#ifndef ZSTD_H_235446 +#define ZSTD_H_235446 + +/* ====== Dependency ======*/ +#include /* size_t */ + + +/* ===== ZSTDLIB_API : control library symbols visibility ===== */ +#if defined(__GNUC__) && (__GNUC__ >= 4) +# define ZSTDLIB_API __attribute__ ((visibility ("default"))) +#elif defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZSTDLIB_API __declspec(dllexport) +#elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZSTDLIB_API __declspec(dllimport) /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +#else +# define ZSTDLIB_API +#endif + + +/******************************************************************************************************* + Introduction + + zstd, short for Zstandard, is a fast lossless compression algorithm, targeting real-time compression scenarios + at zlib-level and better compression ratios. The zstd compression library provides in-memory compression and + decompression functions. The library supports compression levels from 1 up to ZSTD_maxCLevel() which is 22. + Levels >= 20, labelled `--ultra`, should be used with caution, as they require more memory. + Compression can be done in: + - a single step (described as Simple API) + - a single step, reusing a context (described as Explicit memory management) + - unbounded multiple steps (described as Streaming compression) + The compression ratio achievable on small data can be highly improved using compression with a dictionary in: + - a single step (described as Simple dictionary API) + - a single step, reusing a dictionary (described as Fast dictionary API) + + Advanced experimental functions can be accessed using #define ZSTD_STATIC_LINKING_ONLY before including zstd.h. + These APIs shall never be used with a dynamic library. + They are not "stable", their definition may change in the future. Only static linking is allowed. +*********************************************************************************************************/ + +/*------ Version ------*/ +#define ZSTD_VERSION_MAJOR 1 +#define ZSTD_VERSION_MINOR 1 +#define ZSTD_VERSION_RELEASE 2 + +#define ZSTD_LIB_VERSION ZSTD_VERSION_MAJOR.ZSTD_VERSION_MINOR.ZSTD_VERSION_RELEASE +#define ZSTD_QUOTE(str) #str +#define ZSTD_EXPAND_AND_QUOTE(str) ZSTD_QUOTE(str) +#define ZSTD_VERSION_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_LIB_VERSION) + +#define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) +ZSTDLIB_API unsigned ZSTD_versionNumber(void); /**< library version number; to be used when checking dll version */ + + +/*************************************** +* Simple API +***************************************/ +/*! ZSTD_compress() : + Compresses `src` content as a single zstd compressed frame into already allocated `dst`. + Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + @return : compressed size written into `dst` (<= `dstCapacity), + or an error code if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + int compressionLevel); + +/*! ZSTD_decompress() : + `compressedSize` : must be the _exact_ size of a single compressed frame. + `dstCapacity` is an upper bound of originalSize. + If user cannot imply a maximum upper bound, it's better to use streaming mode to decompress data. + @return : the number of bytes decompressed into `dst` (<= `dstCapacity`), + or an errorCode if it fails (which can be tested using ZSTD_isError()). */ +ZSTDLIB_API size_t ZSTD_decompress( void* dst, size_t dstCapacity, + const void* src, size_t compressedSize); + +/*! ZSTD_getDecompressedSize() : +* 'src' is the start of a zstd compressed frame. +* @return : content size to be decompressed, as a 64-bits value _if known_, 0 otherwise. +* note 1 : decompressed size is an optional field, that may not be present, especially in streaming mode. +* When `return==0`, data to decompress could be any size. +* In which case, it's necessary to use streaming mode to decompress data. +* Optionally, application can still use ZSTD_decompress() while relying on implied limits. +* (For example, data may be necessarily cut into blocks <= 16 KB). +* note 2 : decompressed size is always present when compression is done with ZSTD_compress() +* note 3 : decompressed size can be very large (64-bits value), +* potentially larger than what local system can handle as a single memory segment. +* In which case, it's necessary to use streaming mode to decompress data. +* note 4 : If source is untrusted, decompressed size could be wrong or intentionally modified. +* Always ensure result fits within application's authorized limits. +* Each application can set its own limits. +* note 5 : when `return==0`, if precise failure cause is needed, use ZSTD_getFrameParams() to know more. */ +ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); + + +/*====== Helper functions ======*/ +ZSTDLIB_API int ZSTD_maxCLevel(void); /*!< maximum compression level available */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case scenario */ +ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ +ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ + + +/*************************************** +* Explicit memory management +***************************************/ +/*= Compression context +* When compressing many times, +* it is recommended to allocate a context just once, and re-use it for each successive compression operation. +* This will make workload friendlier for system's memory. +* Use one context per thread for parallel execution in multi-threaded environments. */ +typedef struct ZSTD_CCtx_s ZSTD_CCtx; +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx(void); +ZSTDLIB_API size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx); + +/*! ZSTD_compressCCtx() : + Same as ZSTD_compress(), requires an allocated ZSTD_CCtx (see ZSTD_createCCtx()). */ +ZSTDLIB_API size_t ZSTD_compressCCtx(ZSTD_CCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, int compressionLevel); + +/*= Decompression context */ +typedef struct ZSTD_DCtx_s ZSTD_DCtx; +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx(void); +ZSTDLIB_API size_t ZSTD_freeDCtx(ZSTD_DCtx* dctx); + +/*! ZSTD_decompressDCtx() : +* Same as ZSTD_decompress(), requires an allocated ZSTD_DCtx (see ZSTD_createDCtx()). */ +ZSTDLIB_API size_t ZSTD_decompressDCtx(ZSTD_DCtx* ctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + +/************************** +* Simple dictionary API +***************************/ +/*! ZSTD_compress_usingDict() : +* Compression using a predefined Dictionary (see dictBuilder/zdict.h). +* Note : This function loads the dictionary, resulting in significant startup delay. +* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_compress_usingDict(ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + int compressionLevel); + +/*! ZSTD_decompress_usingDict() : +* Decompression using a predefined Dictionary (see dictBuilder/zdict.h). +* Dictionary must be identical to the one used during compression. +* Note : This function loads the dictionary, resulting in significant startup delay. +* Note : When `dict == NULL || dictSize < 8` no dictionary is used. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize); + + +/**************************** +* Fast dictionary API +****************************/ +typedef struct ZSTD_CDict_s ZSTD_CDict; + +/*! ZSTD_createCDict() : +* When compressing multiple messages / blocks with the same dictionary, it's recommended to load it just once. +* ZSTD_createCDict() will create a digested dictionary, ready to start future compression operations without startup delay. +* ZSTD_CDict can be created once and used by multiple threads concurrently, as its usage is read-only. +* `dict` can be released after ZSTD_CDict creation. */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict(const void* dict, size_t dictSize, int compressionLevel); + +/*! ZSTD_freeCDict() : +* Function frees memory allocated by ZSTD_createCDict(). */ +ZSTDLIB_API size_t ZSTD_freeCDict(ZSTD_CDict* CDict); + +/*! ZSTD_compress_usingCDict() : +* Compression using a digested Dictionary. +* Faster startup than ZSTD_compress_usingDict(), recommended when same dictionary is used multiple times. +* Note that compression level is decided during dictionary creation. */ +ZSTDLIB_API size_t ZSTD_compress_usingCDict(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_CDict* cdict); + + +typedef struct ZSTD_DDict_s ZSTD_DDict; + +/*! ZSTD_createDDict() : +* Create a digested dictionary, ready to start decompression operation without startup delay. +* `dict` can be released after creation. */ +ZSTDLIB_API ZSTD_DDict* ZSTD_createDDict(const void* dict, size_t dictSize); + +/*! ZSTD_freeDDict() : +* Function frees memory allocated with ZSTD_createDDict() */ +ZSTDLIB_API size_t ZSTD_freeDDict(ZSTD_DDict* ddict); + +/*! ZSTD_decompress_usingDDict() : +* Decompression using a digested Dictionary. +* Faster startup than ZSTD_decompress_usingDict(), recommended when same dictionary is used multiple times. */ +ZSTDLIB_API size_t ZSTD_decompress_usingDDict(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const ZSTD_DDict* ddict); + + +/**************************** +* Streaming +****************************/ + +typedef struct ZSTD_inBuffer_s { + const void* src; /**< start of input buffer */ + size_t size; /**< size of input buffer */ + size_t pos; /**< position where reading stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_inBuffer; + +typedef struct ZSTD_outBuffer_s { + void* dst; /**< start of output buffer */ + size_t size; /**< size of output buffer */ + size_t pos; /**< position where writing stopped. Will be updated. Necessarily 0 <= pos <= size */ +} ZSTD_outBuffer; + + + +/*-*********************************************************************** +* Streaming compression - HowTo +* +* A ZSTD_CStream object is required to track streaming operation. +* Use ZSTD_createCStream() and ZSTD_freeCStream() to create/release resources. +* ZSTD_CStream objects can be reused multiple times on consecutive compression operations. +* It is recommended to re-use ZSTD_CStream in situations where many streaming operations will be achieved consecutively, +* since it will play nicer with system's memory, by re-using already allocated memory. +* Use one separate ZSTD_CStream per thread for parallel execution. +* +* Start a new compression by initializing ZSTD_CStream. +* Use ZSTD_initCStream() to start a new compression operation. +* Use ZSTD_initCStream_usingDict() or ZSTD_initCStream_usingCDict() for a compression which requires a dictionary (experimental section) +* +* Use ZSTD_compressStream() repetitively to consume input stream. +* The function will automatically update both `pos` fields. +* Note that it may not consume the entire input, in which case `pos < size`, +* and it's up to the caller to present again remaining data. +* @return : a size hint, preferred nb of bytes to use as input for next function call +* or an error code, which can be tested using ZSTD_isError(). +* Note 1 : it's just a hint, to help latency a little, any other value will work fine. +* Note 2 : size hint is guaranteed to be <= ZSTD_CStreamInSize() +* +* At any moment, it's possible to flush whatever data remains within internal buffer, using ZSTD_flushStream(). +* `output->pos` will be updated. +* Note that some content might still be left within internal buffer if `output->size` is too small. +* @return : nb of bytes still present within internal buffer (0 if it's empty) +* or an error code, which can be tested using ZSTD_isError(). +* +* ZSTD_endStream() instructs to finish a frame. +* It will perform a flush and write frame epilogue. +* The epilogue is required for decoders to consider a frame completed. +* Similar to ZSTD_flushStream(), it may not be able to flush the full content if `output->size` is too small. +* In which case, call again ZSTD_endStream() to complete the flush. +* @return : nb of bytes still present within internal buffer (0 if it's empty, hence compression completed) +* or an error code, which can be tested using ZSTD_isError(). +* +* *******************************************************************/ + +typedef struct ZSTD_CStream_s ZSTD_CStream; +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream(void); +ZSTDLIB_API size_t ZSTD_freeCStream(ZSTD_CStream* zcs); + +ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input); +ZSTDLIB_API size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); +ZSTDLIB_API size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output); + +ZSTDLIB_API size_t ZSTD_CStreamInSize(void); /**< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output buffer. Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + + + +/*-*************************************************************************** +* Streaming decompression - HowTo +* +* A ZSTD_DStream object is required to track streaming operations. +* Use ZSTD_createDStream() and ZSTD_freeDStream() to create/release resources. +* ZSTD_DStream objects can be re-used multiple times. +* +* Use ZSTD_initDStream() to start a new decompression operation, +* or ZSTD_initDStream_usingDict() if decompression requires a dictionary. +* @return : recommended first input size +* +* Use ZSTD_decompressStream() repetitively to consume your input. +* The function will update both `pos` fields. +* If `input.pos < input.size`, some input has not been consumed. +* It's up to the caller to present again remaining data. +* If `output.pos < output.size`, decoder has flushed everything it could. +* @return : 0 when a frame is completely decoded and fully flushed, +* an error code, which can be tested using ZSTD_isError(), +* any other value > 0, which means there is still some decoding to do to complete current frame. +* The return value is a suggested next input size (a hint to improve latency) that will never load more than the current frame. +* *******************************************************************************/ + +typedef struct ZSTD_DStream_s ZSTD_DStream; +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream(void); +ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); + +ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); + +ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ +ZSTDLIB_API size_t ZSTD_DStreamOutSize(void); /*!< recommended size for output buffer. Guarantee to successfully flush at least one complete block in all circumstances. */ + +#endif /* ZSTD_H_235446 */ + + +#if defined(ZSTD_STATIC_LINKING_ONLY) && !defined(ZSTD_H_ZSTD_STATIC_LINKING_ONLY) +#define ZSTD_H_ZSTD_STATIC_LINKING_ONLY + +/**************************************************************************************** + * START OF ADVANCED AND EXPERIMENTAL FUNCTIONS + * The definitions in this section are considered experimental. + * They should never be used with a dynamic library, as they may change in the future. + * They are provided for advanced usages. + * Use them only in association with static linking. + * ***************************************************************************************/ + +/* --- Constants ---*/ +#define ZSTD_MAGICNUMBER 0xFD2FB528 /* v0.8 */ +#define ZSTD_MAGIC_SKIPPABLE_START 0x184D2A50U + +#define ZSTD_WINDOWLOG_MAX_32 25 +#define ZSTD_WINDOWLOG_MAX_64 27 +#define ZSTD_WINDOWLOG_MAX ((U32)(MEM_32bits() ? ZSTD_WINDOWLOG_MAX_32 : ZSTD_WINDOWLOG_MAX_64)) +#define ZSTD_WINDOWLOG_MIN 10 +#define ZSTD_HASHLOG_MAX ZSTD_WINDOWLOG_MAX +#define ZSTD_HASHLOG_MIN 6 +#define ZSTD_CHAINLOG_MAX (ZSTD_WINDOWLOG_MAX+1) +#define ZSTD_CHAINLOG_MIN ZSTD_HASHLOG_MIN +#define ZSTD_HASHLOG3_MAX 17 +#define ZSTD_SEARCHLOG_MAX (ZSTD_WINDOWLOG_MAX-1) +#define ZSTD_SEARCHLOG_MIN 1 +#define ZSTD_SEARCHLENGTH_MAX 7 /* only for ZSTD_fast, other strategies are limited to 6 */ +#define ZSTD_SEARCHLENGTH_MIN 3 /* only for ZSTD_btopt, other strategies are limited to 4 */ +#define ZSTD_TARGETLENGTH_MIN 4 +#define ZSTD_TARGETLENGTH_MAX 999 + +#define ZSTD_FRAMEHEADERSIZE_MAX 18 /* for static allocation */ +static const size_t ZSTD_frameHeaderSize_prefix = 5; +static const size_t ZSTD_frameHeaderSize_min = 6; +static const size_t ZSTD_frameHeaderSize_max = ZSTD_FRAMEHEADERSIZE_MAX; +static const size_t ZSTD_skippableHeaderSize = 8; /* magic number + skippable frame length */ + + +/*--- Advanced types ---*/ +typedef enum { ZSTD_fast, ZSTD_dfast, ZSTD_greedy, ZSTD_lazy, ZSTD_lazy2, ZSTD_btlazy2, ZSTD_btopt, ZSTD_btopt2 } ZSTD_strategy; /* from faster to stronger */ + +typedef struct { + unsigned windowLog; /**< largest match distance : larger == more compression, more memory needed during decompression */ + unsigned chainLog; /**< fully searched segment : larger == more compression, slower, more memory (useless for fast) */ + unsigned hashLog; /**< dispatch table : larger == faster, more memory */ + unsigned searchLog; /**< nb of searches : larger == more compression, slower */ + unsigned searchLength; /**< match length searched : larger == faster decompression, sometimes less compression */ + unsigned targetLength; /**< acceptable match size for optimal parser (only) : larger == more compression, slower */ + ZSTD_strategy strategy; +} ZSTD_compressionParameters; + +typedef struct { + unsigned contentSizeFlag; /**< 1: content size will be in frame header (if known). */ + unsigned checksumFlag; /**< 1: will generate a 22-bits checksum at end of frame, to be used for error detection by decompressor */ + unsigned noDictIDFlag; /**< 1: no dict ID will be saved into frame header (if dictionary compression) */ +} ZSTD_frameParameters; + +typedef struct { + ZSTD_compressionParameters cParams; + ZSTD_frameParameters fParams; +} ZSTD_parameters; + +/*= Custom memory allocation functions */ +typedef void* (*ZSTD_allocFunction) (void* opaque, size_t size); +typedef void (*ZSTD_freeFunction) (void* opaque, void* address); +typedef struct { ZSTD_allocFunction customAlloc; ZSTD_freeFunction customFree; void* opaque; } ZSTD_customMem; + + +/*************************************** +* Advanced compression functions +***************************************/ +/*! ZSTD_estimateCCtxSize() : + * Gives the amount of memory allocated for a ZSTD_CCtx given a set of compression parameters. + * `frameContentSize` is an optional parameter, provide `0` if unknown */ +ZSTDLIB_API size_t ZSTD_estimateCCtxSize(ZSTD_compressionParameters cParams); + +/*! ZSTD_createCCtx_advanced() : + * Create a ZSTD compression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_CCtx* ZSTD_createCCtx_advanced(ZSTD_customMem customMem); + +/*! ZSTD_sizeofCCtx() : + * Gives the amount of memory used by a given ZSTD_CCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_CCtx(const ZSTD_CCtx* cctx); + +/*! ZSTD_createCDict_advanced() : + * Create a ZSTD_CDict using external alloc and free, and customized compression parameters */ +ZSTDLIB_API ZSTD_CDict* ZSTD_createCDict_advanced(const void* dict, size_t dictSize, + ZSTD_parameters params, ZSTD_customMem customMem); + +/*! ZSTD_sizeof_CDict() : + * Gives the amount of memory used by a given ZSTD_sizeof_CDict */ +ZSTDLIB_API size_t ZSTD_sizeof_CDict(const ZSTD_CDict* cdict); + +/*! ZSTD_getCParams() : +* @return ZSTD_compressionParameters structure for a selected compression level and estimated srcSize. +* `estimatedSrcSize` value is optional, select 0 if not known */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_getCParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_getParams() : +* same as ZSTD_getCParams(), but @return a full `ZSTD_parameters` object instead of sub-component `ZSTD_compressionParameters`. +* All fields of `ZSTD_frameParameters` are set to default (0) */ +ZSTDLIB_API ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long estimatedSrcSize, size_t dictSize); + +/*! ZSTD_checkCParams() : +* Ensure param values remain within authorized range */ +ZSTDLIB_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); + +/*! ZSTD_adjustCParams() : +* optimize params for a given `srcSize` and `dictSize`. +* both values are optional, select `0` if unknown. */ +ZSTDLIB_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); + +/*! ZSTD_compress_advanced() : +* Same as ZSTD_compress_usingDict(), with fine-tune control of each compression parameter */ +ZSTDLIB_API size_t ZSTD_compress_advanced (ZSTD_CCtx* ctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); + + +/*--- Advanced decompression functions ---*/ + +/*! ZSTD_isFrame() : + * Tells if the content of `buffer` starts with a valid Frame Identifier. + * Note : Frame Identifier is 4 bytes. If `size < 4`, @return will always be 0. + * Note 2 : Legacy Frame Identifiers are considered valid only if Legacy Support is enabled. + * Note 3 : Skippable Frame Identifiers are considered valid. */ +ZSTDLIB_API unsigned ZSTD_isFrame(const void* buffer, size_t size); + +/*! ZSTD_estimateDCtxSize() : + * Gives the potential amount of memory allocated to create a ZSTD_DCtx */ +ZSTDLIB_API size_t ZSTD_estimateDCtxSize(void); + +/*! ZSTD_createDCtx_advanced() : + * Create a ZSTD decompression context using external alloc and free functions */ +ZSTDLIB_API ZSTD_DCtx* ZSTD_createDCtx_advanced(ZSTD_customMem customMem); + +/*! ZSTD_sizeof_DCtx() : + * Gives the amount of memory used by a given ZSTD_DCtx */ +ZSTDLIB_API size_t ZSTD_sizeof_DCtx(const ZSTD_DCtx* dctx); + +/*! ZSTD_sizeof_DDict() : + * Gives the amount of memory used by a given ZSTD_DDict */ +ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromDict() : + * Provides the dictID stored within dictionary. + * if @return == 0, the dictionary is not conformant with Zstandard specification. + * It can still be loaded, but as a content-only dictionary. */ +unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize); + +/*! ZSTD_getDictID_fromDDict() : + * Provides the dictID of the dictionary loaded into `ddict`. + * If @return == 0, the dictionary is not conformant to Zstandard specification, or empty. + * Non-conformant dictionaries can still be loaded, but as content-only dictionaries. */ +unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); + +/*! ZSTD_getDictID_fromFrame() : + * Provides the dictID required to decompressed the frame stored within `src`. + * If @return == 0, the dictID could not be decoded. + * This could for one of the following reasons : + * - The frame does not require a dictionary to be decoded (most common case). + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * Note : this use case also happens when using a non-conformant dictionary. + * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). + * - This is not a Zstandard frame. + * When identifying the exact failure cause, it's possible to used ZSTD_getFrameParams(), which will provide a more precise error code. */ +unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); + + +/******************************************************************** +* Advanced streaming functions +********************************************************************/ + +/*===== Advanced Streaming compression functions =====*/ +ZSTDLIB_API ZSTD_CStream* ZSTD_createCStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); /**< pledgedSrcSize must be correct */ +ZSTDLIB_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, + ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize is optional and can be zero == unknown */ +ZSTDLIB_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /**< note : cdict will just be referenced, and must outlive compression session */ +ZSTDLIB_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); /**< re-use compression parameters from previous init; skip dictionary loading stage; zcs must be init at least once before */ +ZSTDLIB_API size_t ZSTD_sizeof_CStream(const ZSTD_CStream* zcs); + + +/*===== Advanced Streaming decompression functions =====*/ +typedef enum { ZSTDdsp_maxWindowSize } ZSTD_DStreamParameter_e; +ZSTDLIB_API ZSTD_DStream* ZSTD_createDStream_advanced(ZSTD_customMem customMem); +ZSTDLIB_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); +ZSTDLIB_API size_t ZSTD_setDStreamParameter(ZSTD_DStream* zds, ZSTD_DStreamParameter_e paramType, unsigned paramValue); +ZSTDLIB_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /**< note : ddict will just be referenced, and must outlive decompression session */ +ZSTDLIB_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); /**< re-use decompression parameters from previous init; saves dictionary loading */ +ZSTDLIB_API size_t ZSTD_sizeof_DStream(const ZSTD_DStream* zds); + + +/********************************************************************* +* Buffer-less and synchronous inner streaming functions +* +* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. +* But it's also a complex one, with many restrictions (documented below). +* Prefer using normal streaming API for an easier experience +********************************************************************* */ + +/** + Buffer-less streaming compression (synchronous mode) + + A ZSTD_CCtx object is required to track streaming operations. + Use ZSTD_createCCtx() / ZSTD_freeCCtx() to manage resource. + ZSTD_CCtx object can be re-used multiple times within successive compression operations. + + Start by initializing a context. + Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression, + or ZSTD_compressBegin_advanced(), for finer parameter control. + It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() + + Then, consume your input using ZSTD_compressContinue(). + There are some important considerations to keep in mind when using this advanced function : + - ZSTD_compressContinue() has no internal buffer. It uses externally provided buffer only. + - Interface is synchronous : input is consumed entirely and produce 1+ (or more) compressed blocks. + - Caller must ensure there is enough space in `dst` to store compressed data under worst case scenario. + Worst case evaluation is provided by ZSTD_compressBound(). + ZSTD_compressContinue() doesn't guarantee recover after a failed compression. + - ZSTD_compressContinue() presumes prior input ***is still accessible and unmodified*** (up to maximum distance size, see WindowLog). + It remembers all previous contiguous blocks, plus one separated memory segment (which can itself consists of multiple contiguous blocks) + - ZSTD_compressContinue() detects that prior input has been overwritten when `src` buffer overlaps. + In which case, it will "discard" the relevant memory section from its history. + + Finish a frame with ZSTD_compressEnd(), which will write the last block(s) and optional checksum. + It's possible to use a NULL,0 src content, in which case, it will write a final empty block to end the frame, + Without last block mark, frames will be considered unfinished (broken) by decoders. + + You can then reuse `ZSTD_CCtx` (ZSTD_compressBegin()) to compress some new frame. +*/ + +/*===== Buffer-less streaming compression functions =====*/ +ZSTDLIB_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTDLIB_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); +ZSTDLIB_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + + +/*- + Buffer-less streaming decompression (synchronous mode) + + A ZSTD_DCtx object is required to track streaming operations. + Use ZSTD_createDCtx() / ZSTD_freeDCtx() to manage it. + A ZSTD_DCtx object can be re-used multiple times. + + First typical operation is to retrieve frame parameters, using ZSTD_getFrameParams(). + It fills a ZSTD_frameParams structure which provide important information to correctly decode the frame, + such as the minimum rolling buffer size to allocate to decompress data (`windowSize`), + and the dictionary ID used. + (Note : content size is optional, it may not be present. 0 means : content size unknown). + Note that these values could be wrong, either because of data malformation, or because an attacker is spoofing deliberate false information. + As a consequence, check that values remain within valid application range, especially `windowSize`, before allocation. + Each application can set its own limit, depending on local restrictions. For extended interoperability, it is recommended to support at least 8 MB. + Frame parameters are extracted from the beginning of the compressed frame. + Data fragment must be large enough to ensure successful decoding, typically `ZSTD_frameHeaderSize_max` bytes. + @result : 0 : successful decoding, the `ZSTD_frameParams` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + errorCode, which can be tested using ZSTD_isError(). + + Start decompression, with ZSTD_decompressBegin() or ZSTD_decompressBegin_usingDict(). + Alternatively, you can copy a prepared context, using ZSTD_copyDCtx(). + + Then use ZSTD_nextSrcSizeToDecompress() and ZSTD_decompressContinue() alternatively. + ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). + ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. + + @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + It can be zero, which is not an error; it just means ZSTD_decompressContinue() has decoded some metadata item. + It can also be an error code, which can be tested with ZSTD_isError(). + + ZSTD_decompressContinue() needs previous data blocks during decompression, up to `windowSize`. + They should preferably be located contiguously, prior to current block. + Alternatively, a round buffer of sufficient size is also possible. Sufficient size is determined by frame parameters. + ZSTD_decompressContinue() is very sensitive to contiguity, + if 2 blocks don't follow each other, make sure that either the compressor breaks contiguity at the same place, + or that previous contiguous segment is large enough to properly handle maximum back-reference. + + A frame is fully decoded when ZSTD_nextSrcSizeToDecompress() returns zero. + Context can then be reset to start a new decompression. + + Note : it's possible to know if next input to present is a header or a block, using ZSTD_nextInputType(). + This information is not required to properly decode a frame. + + == Special case : skippable frames == + + Skippable frames allow integration of user-defined data into a flow of concatenated frames. + Skippable frames will be ignored (skipped) by a decompressor. The format of skippable frames is as follows : + a) Skippable frame ID - 4 Bytes, Little endian format, any value from 0x184D2A50 to 0x184D2A5F + b) Frame Size - 4 Bytes, Little endian format, unsigned 32-bits + c) Frame Content - any content (User Data) of length equal to Frame Size + For skippable frames ZSTD_decompressContinue() always returns 0. + For skippable frames ZSTD_getFrameParams() returns fparamsPtr->windowLog==0 what means that a frame is skippable. + It also returns Frame Size as fparamsPtr->frameContentSize. +*/ + +typedef struct { + unsigned long long frameContentSize; + unsigned windowSize; + unsigned dictID; + unsigned checksumFlag; +} ZSTD_frameParams; + +/*===== Buffer-less streaming decompression functions =====*/ +ZSTDLIB_API size_t ZSTD_getFrameParams(ZSTD_frameParams* fparamsPtr, const void* src, size_t srcSize); /**< doesn't consume input, see details below */ +ZSTDLIB_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressBegin_usingDict(ZSTD_DCtx* dctx, const void* dict, size_t dictSize); +ZSTDLIB_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); +ZSTDLIB_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); +ZSTDLIB_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; +ZSTDLIB_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); + +/** + Block functions + + Block functions produce and decode raw zstd blocks, without frame metadata. + Frame metadata cost is typically ~18 bytes, which can be non-negligible for very small blocks (< 100 bytes). + User will have to take in charge required information to regenerate data, such as compressed and content sizes. + + A few rules to respect : + - Compressing and decompressing require a context structure + + Use ZSTD_createCCtx() and ZSTD_createDCtx() + - It is necessary to init context before starting + + compression : ZSTD_compressBegin() + + decompression : ZSTD_decompressBegin() + + variants _usingDict() are also allowed + + copyCCtx() and copyDCtx() work too + - Block size is limited, it must be <= ZSTD_getBlockSizeMax() + + If you need to compress more, cut data into multiple blocks + + Consider using the regular ZSTD_compress() instead, as frame metadata costs become negligible when source size is large. + - When a block is considered not compressible enough, ZSTD_compressBlock() result will be zero. + In which case, nothing is produced into `dst`. + + User must test for such outcome and deal directly with uncompressed data + + ZSTD_decompressBlock() doesn't accept uncompressed data as input !!! + + In case of multiple successive blocks, decoder must be informed of uncompressed block existence to follow proper history. + Use ZSTD_insertBlock() in such a case. +*/ + +#define ZSTD_BLOCKSIZE_ABSOLUTEMAX (128 * 1024) /* define, for static allocation */ +/*===== Raw zstd block functions =====*/ +ZSTDLIB_API size_t ZSTD_getBlockSizeMax(ZSTD_CCtx* cctx); +ZSTDLIB_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTDLIB_API size_t ZSTD_insertBlock(ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert block into `dctx` history. Useful for uncompressed blocks */ + + +#endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ + +#if defined (__cplusplus) +} +#endif diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd.c mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd.c --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd.c 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd.c 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,136 @@ +/** + * Copyright (c) 2016-present, Gregory Szorc + * All rights reserved. + * + * This software may be modified and distributed under the terms + * of the BSD license. See the LICENSE file for details. + */ + +/* A Python C extension for Zstandard. */ + +#include "python-zstandard.h" + +PyObject *ZstdError; + +PyDoc_STRVAR(estimate_compression_context_size__doc__, +"estimate_compression_context_size(compression_parameters)\n" +"\n" +"Give the amount of memory allocated for a compression context given a\n" +"CompressionParameters instance"); + +PyDoc_STRVAR(estimate_decompression_context_size__doc__, +"estimate_decompression_context_size()\n" +"\n" +"Estimate the amount of memory allocated to a decompression context.\n" +); + +static PyObject* estimate_decompression_context_size(PyObject* self) { + return PyLong_FromSize_t(ZSTD_estimateDCtxSize()); +} + +PyDoc_STRVAR(get_compression_parameters__doc__, +"get_compression_parameters(compression_level[, source_size[, dict_size]])\n" +"\n" +"Obtains a ``CompressionParameters`` instance from a compression level and\n" +"optional input size and dictionary size"); + +PyDoc_STRVAR(train_dictionary__doc__, +"train_dictionary(dict_size, samples)\n" +"\n" +"Train a dictionary from sample data.\n" +"\n" +"A compression dictionary of size ``dict_size`` will be created from the\n" +"iterable of samples provided by ``samples``.\n" +"\n" +"The raw dictionary content will be returned\n"); + +static char zstd_doc[] = "Interface to zstandard"; + +static PyMethodDef zstd_methods[] = { + { "estimate_compression_context_size", (PyCFunction)estimate_compression_context_size, + METH_VARARGS, estimate_compression_context_size__doc__ }, + { "estimate_decompression_context_size", (PyCFunction)estimate_decompression_context_size, + METH_NOARGS, estimate_decompression_context_size__doc__ }, + { "get_compression_parameters", (PyCFunction)get_compression_parameters, + METH_VARARGS, get_compression_parameters__doc__ }, + { "train_dictionary", (PyCFunction)train_dictionary, + METH_VARARGS | METH_KEYWORDS, train_dictionary__doc__ }, + { NULL, NULL } +}; + +void compressobj_module_init(PyObject* mod); +void compressor_module_init(PyObject* mod); +void compressionparams_module_init(PyObject* mod); +void constants_module_init(PyObject* mod); +void dictparams_module_init(PyObject* mod); +void compressiondict_module_init(PyObject* mod); +void compressionwriter_module_init(PyObject* mod); +void compressoriterator_module_init(PyObject* mod); +void decompressor_module_init(PyObject* mod); +void decompressobj_module_init(PyObject* mod); +void decompressionwriter_module_init(PyObject* mod); +void decompressoriterator_module_init(PyObject* mod); + +void zstd_module_init(PyObject* m) { + /* python-zstandard relies on unstable zstd C API features. This means + that changes in zstd may break expectations in python-zstandard. + + python-zstandard is distributed with a copy of the zstd sources. + python-zstandard is only guaranteed to work with the bundled version + of zstd. + + However, downstream redistributors or packagers may unbundle zstd + from python-zstandard. This can result in a mismatch between zstd + versions and API semantics. This essentially "voids the warranty" + of python-zstandard and may cause undefined behavior. + + We detect this mismatch here and refuse to load the module if this + scenario is detected. + */ + if (ZSTD_VERSION_NUMBER != 10102 || ZSTD_versionNumber() != 10102) { + PyErr_SetString(PyExc_ImportError, "zstd C API mismatch; Python bindings not compiled against expected zstd version"); + return; + } + + compressionparams_module_init(m); + dictparams_module_init(m); + compressiondict_module_init(m); + compressobj_module_init(m); + compressor_module_init(m); + compressionwriter_module_init(m); + compressoriterator_module_init(m); + constants_module_init(m); + decompressor_module_init(m); + decompressobj_module_init(m); + decompressionwriter_module_init(m); + decompressoriterator_module_init(m); +} + +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef zstd_module = { + PyModuleDef_HEAD_INIT, + "zstd", + zstd_doc, + -1, + zstd_methods +}; + +PyMODINIT_FUNC PyInit_zstd(void) { + PyObject *m = PyModule_Create(&zstd_module); + if (m) { + zstd_module_init(m); + if (PyErr_Occurred()) { + Py_DECREF(m); + m = NULL; + } + } + return m; +} +#else +PyMODINIT_FUNC initzstd(void) { + PyObject *m = Py_InitModule3("zstd", zstd_methods, zstd_doc); + if (m) { + zstd_module_init(m); + } +} +#endif diff -Nru mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd_cffi.py mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd_cffi.py --- mercurial-4.0.2~trusty1/contrib/python-zstandard/zstd_cffi.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/python-zstandard/zstd_cffi.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,152 @@ +# Copyright (c) 2016-present, Gregory Szorc +# All rights reserved. +# +# This software may be modified and distributed under the terms +# of the BSD license. See the LICENSE file for details. + +"""Python interface to the Zstandard (zstd) compression library.""" + +from __future__ import absolute_import, unicode_literals + +import io + +from _zstd_cffi import ( + ffi, + lib, +) + + +_CSTREAM_IN_SIZE = lib.ZSTD_CStreamInSize() +_CSTREAM_OUT_SIZE = lib.ZSTD_CStreamOutSize() + + +class _ZstdCompressionWriter(object): + def __init__(self, cstream, writer): + self._cstream = cstream + self._writer = writer + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + if not exc_type and not exc_value and not exc_tb: + out_buffer = ffi.new('ZSTD_outBuffer *') + out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE) + out_buffer.size = _CSTREAM_OUT_SIZE + out_buffer.pos = 0 + + while True: + res = lib.ZSTD_endStream(self._cstream, out_buffer) + if lib.ZSTD_isError(res): + raise Exception('error ending compression stream: %s' % lib.ZSTD_getErrorName) + + if out_buffer.pos: + self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + out_buffer.pos = 0 + + if res == 0: + break + + return False + + def write(self, data): + out_buffer = ffi.new('ZSTD_outBuffer *') + out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE) + out_buffer.size = _CSTREAM_OUT_SIZE + out_buffer.pos = 0 + + # TODO can we reuse existing memory? + in_buffer = ffi.new('ZSTD_inBuffer *') + in_buffer.src = ffi.new('char[]', data) + in_buffer.size = len(data) + in_buffer.pos = 0 + while in_buffer.pos < in_buffer.size: + res = lib.ZSTD_compressStream(self._cstream, out_buffer, in_buffer) + if lib.ZSTD_isError(res): + raise Exception('zstd compress error: %s' % lib.ZSTD_getErrorName(res)) + + if out_buffer.pos: + self._writer.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + out_buffer.pos = 0 + + +class ZstdCompressor(object): + def __init__(self, level=3, dict_data=None, compression_params=None): + if dict_data: + raise Exception('dict_data not yet supported') + if compression_params: + raise Exception('compression_params not yet supported') + + self._compression_level = level + + def compress(self, data): + # Just use the stream API for now. + output = io.BytesIO() + with self.write_to(output) as compressor: + compressor.write(data) + return output.getvalue() + + def copy_stream(self, ifh, ofh): + cstream = self._get_cstream() + + in_buffer = ffi.new('ZSTD_inBuffer *') + out_buffer = ffi.new('ZSTD_outBuffer *') + + out_buffer.dst = ffi.new('char[]', _CSTREAM_OUT_SIZE) + out_buffer.size = _CSTREAM_OUT_SIZE + out_buffer.pos = 0 + + total_read, total_write = 0, 0 + + while True: + data = ifh.read(_CSTREAM_IN_SIZE) + if not data: + break + + total_read += len(data) + + in_buffer.src = ffi.new('char[]', data) + in_buffer.size = len(data) + in_buffer.pos = 0 + + while in_buffer.pos < in_buffer.size: + res = lib.ZSTD_compressStream(cstream, out_buffer, in_buffer) + if lib.ZSTD_isError(res): + raise Exception('zstd compress error: %s' % + lib.ZSTD_getErrorName(res)) + + if out_buffer.pos: + ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + total_write = out_buffer.pos + out_buffer.pos = 0 + + # We've finished reading. Flush the compressor. + while True: + res = lib.ZSTD_endStream(cstream, out_buffer) + if lib.ZSTD_isError(res): + raise Exception('error ending compression stream: %s' % + lib.ZSTD_getErrorName(res)) + + if out_buffer.pos: + ofh.write(ffi.buffer(out_buffer.dst, out_buffer.pos)) + total_write += out_buffer.pos + out_buffer.pos = 0 + + if res == 0: + break + + return total_read, total_write + + def write_to(self, writer): + return _ZstdCompressionWriter(self._get_cstream(), writer) + + def _get_cstream(self): + cstream = lib.ZSTD_createCStream() + cstream = ffi.gc(cstream, lib.ZSTD_freeCStream) + + res = lib.ZSTD_initCStream(cstream, self._compression_level) + if lib.ZSTD_isError(res): + raise Exception('cannot init CStream: %s' % + lib.ZSTD_getErrorName(res)) + + return cstream diff -Nru mercurial-4.0.2~trusty1/contrib/simplemerge mercurial-4.1~rc~trusty1/contrib/simplemerge --- mercurial-4.0.2~trusty1/contrib/simplemerge 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/simplemerge 2017-01-18 16:54:51.000000000 +0000 @@ -3,6 +3,7 @@ from mercurial import demandimport demandimport.enable() +import getopt import sys from mercurial.i18n import _ from mercurial import error, simplemerge, fancyopts, util, ui @@ -47,14 +48,14 @@ opts = {} try: args = fancyopts.fancyopts(sys.argv[1:], options, opts) - except fancyopts.getopt.GetoptError as e: + except getopt.GetoptError as e: raise ParseError(e) if opts['help']: showhelp() sys.exit(0) if len(args) != 3: raise ParseError(_('wrong number of arguments')) - sys.exit(simplemerge.simplemerge(ui.ui(), *args, **opts)) + sys.exit(simplemerge.simplemerge(ui.ui.load(), *args, **opts)) except ParseError as e: sys.stdout.write("%s: %s\n" % (sys.argv[0], e)) showhelp() diff -Nru mercurial-4.0.2~trusty1/contrib/vim/patchreview.vim mercurial-4.1~rc~trusty1/contrib/vim/patchreview.vim --- mercurial-4.0.2~trusty1/contrib/vim/patchreview.vim 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/vim/patchreview.vim 2017-01-18 16:54:51.000000000 +0000 @@ -720,7 +720,7 @@ let s:origtabpagenr = tabpagenr() silent! exe 'tabedit ' . StrippedRelativeFilePath if exists('patchcmd') - " modelines in loaded files mess with diff comparision + " modelines in loaded files mess with diff comparison let s:keep_modeline=&modeline let &modeline=0 silent! exe 'vert diffsplit ' . tmpname . '.file' diff -Nru mercurial-4.0.2~trusty1/contrib/wix/help.wxs mercurial-4.1~rc~trusty1/contrib/wix/help.wxs --- mercurial-4.0.2~trusty1/contrib/wix/help.wxs 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/contrib/wix/help.wxs 2017-01-18 16:54:51.000000000 +0000 @@ -25,11 +25,9 @@ - - diff -Nru mercurial-4.0.2~trusty1/debian/changelog mercurial-4.1~rc~trusty1/debian/changelog --- mercurial-4.0.2~trusty1/debian/changelog 2017-01-19 01:13:12.000000000 +0000 +++ mercurial-4.1~rc~trusty1/debian/changelog 2017-01-19 01:13:13.000000000 +0000 @@ -1,5 +1,5 @@ -mercurial (4.0.2~trusty1) trusty; urgency=medium +mercurial (4.1~rc~trusty1) trusty; urgency=medium * Automated build performed by upstream. - -- Mercurial Devel Wed, 04 Jan 2017 16:59:40 +0000 + -- Mercurial Devel Thu, 19 Jan 2017 01:07:16 +0000 diff -Nru mercurial-4.0.2~trusty1/doc/check-seclevel.py mercurial-4.1~rc~trusty1/doc/check-seclevel.py --- mercurial-4.0.2~trusty1/doc/check-seclevel.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/doc/check-seclevel.py 2017-01-18 16:54:51.000000000 +0000 @@ -158,7 +158,7 @@ (options, args) = optparser.parse_args() - ui = uimod.ui() + ui = uimod.ui.load() ui.setconfig('ui', 'verbose', options.verbose, '--verbose') ui.setconfig('ui', 'debug', options.debug, '--debug') diff -Nru mercurial-4.0.2~trusty1/doc/gendoc.py mercurial-4.1~rc~trusty1/doc/gendoc.py --- mercurial-4.0.2~trusty1/doc/gendoc.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/doc/gendoc.py 2017-01-18 16:54:51.000000000 +0000 @@ -217,7 +217,7 @@ if len(sys.argv) > 1: doc = sys.argv[1] - ui = uimod.ui() + ui = uimod.ui.load() if doc == 'hg.1.gendoc': showdoc(ui) else: diff -Nru mercurial-4.0.2~trusty1/.hg_archival.txt mercurial-4.1~rc~trusty1/.hg_archival.txt --- mercurial-4.0.2~trusty1/.hg_archival.txt 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/.hg_archival.txt 2017-01-18 16:54:51.000000000 +0000 @@ -1,4 +1,4 @@ repo: 9117c6561b0bd7792fa13b50d28239d51b78e51f -node: e69874dc1f4e142746ff3df91e678a09c6fc208c +node: a1dd2c0c479e0550040542e392e87bc91262517e branch: stable -tag: 4.0.2 +tag: 4.1-rc diff -Nru mercurial-4.0.2~trusty1/hgext/bugzilla.py mercurial-4.1~rc~trusty1/hgext/bugzilla.py --- mercurial-4.0.2~trusty1/hgext/bugzilla.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/bugzilla.py 2017-01-18 16:54:51.000000000 +0000 @@ -571,9 +571,9 @@ self.send_user_agent(h) self.send_content(h, request_body) - # Deal with differences between Python 2.4-2.6 and 2.7. + # Deal with differences between Python 2.6 and 2.7. # In the former h is a HTTP(S). In the latter it's a - # HTTP(S)Connection. Luckily, the 2.4-2.6 implementation of + # HTTP(S)Connection. Luckily, the 2.6 implementation of # HTTP(S) has an underlying HTTP(S)Connection, so extract # that and use it. try: diff -Nru mercurial-4.0.2~trusty1/hgext/chgserver.py mercurial-4.1~rc~trusty1/hgext/chgserver.py --- mercurial-4.0.2~trusty1/hgext/chgserver.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/chgserver.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,643 +0,0 @@ -# chgserver.py - command server extension for cHg -# -# Copyright 2011 Yuya Nishihara -# -# This software may be used and distributed according to the terms of the -# GNU General Public License version 2 or any later version. - -"""command server extension for cHg (EXPERIMENTAL) - -'S' channel (read/write) - propagate ui.system() request to client - -'attachio' command - attach client's stdio passed by sendmsg() - -'chdir' command - change current directory - -'getpager' command - checks if pager is enabled and which pager should be executed - -'setenv' command - replace os.environ completely - -'setumask' command - set umask - -'validate' command - reload the config and check if the server is up to date - -Config ------- - -:: - - [chgserver] - idletimeout = 3600 # seconds, after which an idle server will exit - skiphash = False # whether to skip config or env change checks -""" - -from __future__ import absolute_import - -import errno -import hashlib -import inspect -import os -import re -import signal -import struct -import sys -import time - -from mercurial.i18n import _ - -from mercurial import ( - cmdutil, - commands, - commandserver, - dispatch, - error, - extensions, - osutil, - util, -) - -# Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for -# extensions which SHIP WITH MERCURIAL. Non-mainline extensions should -# be specifying the version(s) of Mercurial they are tested with, or -# leave the attribute unspecified. -testedwith = 'ships-with-hg-core' - -_log = commandserver.log - -def _hashlist(items): - """return sha1 hexdigest for a list""" - return hashlib.sha1(str(items)).hexdigest() - -# sensitive config sections affecting confighash -_configsections = [ - 'alias', # affects global state commands.table - 'extdiff', # uisetup will register new commands - 'extensions', -] - -# sensitive environment variables affecting confighash -_envre = re.compile(r'''\A(?: - CHGHG - |HG.* - |LANG(?:UAGE)? - |LC_.* - |LD_.* - |PATH - |PYTHON.* - |TERM(?:INFO)? - |TZ - )\Z''', re.X) - -def _confighash(ui): - """return a quick hash for detecting config/env changes - - confighash is the hash of sensitive config items and environment variables. - - for chgserver, it is designed that once confighash changes, the server is - not qualified to serve its client and should redirect the client to a new - server. different from mtimehash, confighash change will not mark the - server outdated and exit since the user can have different configs at the - same time. - """ - sectionitems = [] - for section in _configsections: - sectionitems.append(ui.configitems(section)) - sectionhash = _hashlist(sectionitems) - envitems = [(k, v) for k, v in os.environ.iteritems() if _envre.match(k)] - envhash = _hashlist(sorted(envitems)) - return sectionhash[:6] + envhash[:6] - -def _getmtimepaths(ui): - """get a list of paths that should be checked to detect change - - The list will include: - - extensions (will not cover all files for complex extensions) - - mercurial/__version__.py - - python binary - """ - modules = [m for n, m in extensions.extensions(ui)] - try: - from mercurial import __version__ - modules.append(__version__) - except ImportError: - pass - files = [sys.executable] - for m in modules: - try: - files.append(inspect.getabsfile(m)) - except TypeError: - pass - return sorted(set(files)) - -def _mtimehash(paths): - """return a quick hash for detecting file changes - - mtimehash calls stat on given paths and calculate a hash based on size and - mtime of each file. mtimehash does not read file content because reading is - expensive. therefore it's not 100% reliable for detecting content changes. - it's possible to return different hashes for same file contents. - it's also possible to return a same hash for different file contents for - some carefully crafted situation. - - for chgserver, it is designed that once mtimehash changes, the server is - considered outdated immediately and should no longer provide service. - - mtimehash is not included in confighash because we only know the paths of - extensions after importing them (there is imp.find_module but that faces - race conditions). We need to calculate confighash without importing. - """ - def trystat(path): - try: - st = os.stat(path) - return (st.st_mtime, st.st_size) - except OSError: - # could be ENOENT, EPERM etc. not fatal in any case - pass - return _hashlist(map(trystat, paths))[:12] - -class hashstate(object): - """a structure storing confighash, mtimehash, paths used for mtimehash""" - def __init__(self, confighash, mtimehash, mtimepaths): - self.confighash = confighash - self.mtimehash = mtimehash - self.mtimepaths = mtimepaths - - @staticmethod - def fromui(ui, mtimepaths=None): - if mtimepaths is None: - mtimepaths = _getmtimepaths(ui) - confighash = _confighash(ui) - mtimehash = _mtimehash(mtimepaths) - _log('confighash = %s mtimehash = %s\n' % (confighash, mtimehash)) - return hashstate(confighash, mtimehash, mtimepaths) - -# copied from hgext/pager.py:uisetup() -def _setuppagercmd(ui, options, cmd): - if not ui.formatted(): - return - - p = ui.config("pager", "pager", os.environ.get("PAGER")) - usepager = False - always = util.parsebool(options['pager']) - auto = options['pager'] == 'auto' - - if not p: - pass - elif always: - usepager = True - elif not auto: - usepager = False - else: - attended = ['annotate', 'cat', 'diff', 'export', 'glog', 'log', 'qdiff'] - attend = ui.configlist('pager', 'attend', attended) - ignore = ui.configlist('pager', 'ignore') - cmds, _ = cmdutil.findcmd(cmd, commands.table) - - for cmd in cmds: - var = 'attend-%s' % cmd - if ui.config('pager', var): - usepager = ui.configbool('pager', var) - break - if (cmd in attend or - (cmd not in ignore and not attend)): - usepager = True - break - - if usepager: - ui.setconfig('ui', 'formatted', ui.formatted(), 'pager') - ui.setconfig('ui', 'interactive', False, 'pager') - return p - -def _newchgui(srcui, csystem): - class chgui(srcui.__class__): - def __init__(self, src=None): - super(chgui, self).__init__(src) - if src: - self._csystem = getattr(src, '_csystem', csystem) - else: - self._csystem = csystem - - def system(self, cmd, environ=None, cwd=None, onerr=None, - errprefix=None): - # fallback to the original system method if the output needs to be - # captured (to self._buffers), or the output stream is not stdout - # (e.g. stderr, cStringIO), because the chg client is not aware of - # these situations and will behave differently (write to stdout). - if (any(s[1] for s in self._bufferstates) - or not util.safehasattr(self.fout, 'fileno') - or self.fout.fileno() != sys.stdout.fileno()): - return super(chgui, self).system(cmd, environ, cwd, onerr, - errprefix) - # copied from mercurial/util.py:system() - self.flush() - def py2shell(val): - if val is None or val is False: - return '0' - if val is True: - return '1' - return str(val) - env = os.environ.copy() - if environ: - env.update((k, py2shell(v)) for k, v in environ.iteritems()) - env['HG'] = util.hgexecutable() - rc = self._csystem(cmd, env, cwd) - if rc and onerr: - errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]), - util.explainexit(rc)[0]) - if errprefix: - errmsg = '%s: %s' % (errprefix, errmsg) - raise onerr(errmsg) - return rc - - return chgui(srcui) - -def _loadnewui(srcui, args): - newui = srcui.__class__() - for a in ['fin', 'fout', 'ferr', 'environ']: - setattr(newui, a, getattr(srcui, a)) - if util.safehasattr(srcui, '_csystem'): - newui._csystem = srcui._csystem - - # internal config: extensions.chgserver - newui.setconfig('extensions', 'chgserver', - srcui.config('extensions', 'chgserver'), '--config') - - # command line args - args = args[:] - dispatch._parseconfig(newui, dispatch._earlygetopt(['--config'], args)) - - # stolen from tortoisehg.util.copydynamicconfig() - for section, name, value in srcui.walkconfig(): - source = srcui.configsource(section, name) - if ':' in source or source == '--config': - # path:line or command line - continue - if source == 'none': - # ui.configsource returns 'none' by default - source = '' - newui.setconfig(section, name, value, source) - - # load wd and repo config, copied from dispatch.py - cwds = dispatch._earlygetopt(['--cwd'], args) - cwd = cwds and os.path.realpath(cwds[-1]) or None - rpath = dispatch._earlygetopt(["-R", "--repository", "--repo"], args) - path, newlui = dispatch._getlocal(newui, rpath, wd=cwd) - - return (newui, newlui) - -class channeledsystem(object): - """Propagate ui.system() request in the following format: - - payload length (unsigned int), - cmd, '\0', - cwd, '\0', - envkey, '=', val, '\0', - ... - envkey, '=', val - - and waits: - - exitcode length (unsigned int), - exitcode (int) - """ - def __init__(self, in_, out, channel): - self.in_ = in_ - self.out = out - self.channel = channel - - def __call__(self, cmd, environ, cwd): - args = [util.quotecommand(cmd), os.path.abspath(cwd or '.')] - args.extend('%s=%s' % (k, v) for k, v in environ.iteritems()) - data = '\0'.join(args) - self.out.write(struct.pack('>cI', self.channel, len(data))) - self.out.write(data) - self.out.flush() - - length = self.in_.read(4) - length, = struct.unpack('>I', length) - if length != 4: - raise error.Abort(_('invalid response')) - rc, = struct.unpack('>i', self.in_.read(4)) - return rc - -_iochannels = [ - # server.ch, ui.fp, mode - ('cin', 'fin', 'rb'), - ('cout', 'fout', 'wb'), - ('cerr', 'ferr', 'wb'), -] - -class chgcmdserver(commandserver.server): - def __init__(self, ui, repo, fin, fout, sock, hashstate, baseaddress): - super(chgcmdserver, self).__init__( - _newchgui(ui, channeledsystem(fin, fout, 'S')), repo, fin, fout) - self.clientsock = sock - self._oldios = [] # original (self.ch, ui.fp, fd) before "attachio" - self.hashstate = hashstate - self.baseaddress = baseaddress - if hashstate is not None: - self.capabilities = self.capabilities.copy() - self.capabilities['validate'] = chgcmdserver.validate - - def cleanup(self): - super(chgcmdserver, self).cleanup() - # dispatch._runcatch() does not flush outputs if exception is not - # handled by dispatch._dispatch() - self.ui.flush() - self._restoreio() - - def attachio(self): - """Attach to client's stdio passed via unix domain socket; all - channels except cresult will no longer be used - """ - # tell client to sendmsg() with 1-byte payload, which makes it - # distinctive from "attachio\n" command consumed by client.read() - self.clientsock.sendall(struct.pack('>cI', 'I', 1)) - clientfds = osutil.recvfds(self.clientsock.fileno()) - _log('received fds: %r\n' % clientfds) - - ui = self.ui - ui.flush() - first = self._saveio() - for fd, (cn, fn, mode) in zip(clientfds, _iochannels): - assert fd > 0 - fp = getattr(ui, fn) - os.dup2(fd, fp.fileno()) - os.close(fd) - if not first: - continue - # reset buffering mode when client is first attached. as we want - # to see output immediately on pager, the mode stays unchanged - # when client re-attached. ferr is unchanged because it should - # be unbuffered no matter if it is a tty or not. - if fn == 'ferr': - newfp = fp - else: - # make it line buffered explicitly because the default is - # decided on first write(), where fout could be a pager. - if fp.isatty(): - bufsize = 1 # line buffered - else: - bufsize = -1 # system default - newfp = os.fdopen(fp.fileno(), mode, bufsize) - setattr(ui, fn, newfp) - setattr(self, cn, newfp) - - self.cresult.write(struct.pack('>i', len(clientfds))) - - def _saveio(self): - if self._oldios: - return False - ui = self.ui - for cn, fn, _mode in _iochannels: - ch = getattr(self, cn) - fp = getattr(ui, fn) - fd = os.dup(fp.fileno()) - self._oldios.append((ch, fp, fd)) - return True - - def _restoreio(self): - ui = self.ui - for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels): - newfp = getattr(ui, fn) - # close newfp while it's associated with client; otherwise it - # would be closed when newfp is deleted - if newfp is not fp: - newfp.close() - # restore original fd: fp is open again - os.dup2(fd, fp.fileno()) - os.close(fd) - setattr(self, cn, ch) - setattr(ui, fn, fp) - del self._oldios[:] - - def validate(self): - """Reload the config and check if the server is up to date - - Read a list of '\0' separated arguments. - Write a non-empty list of '\0' separated instruction strings or '\0' - if the list is empty. - An instruction string could be either: - - "unlink $path", the client should unlink the path to stop the - outdated server. - - "redirect $path", the client should attempt to connect to $path - first. If it does not work, start a new server. It implies - "reconnect". - - "exit $n", the client should exit directly with code n. - This may happen if we cannot parse the config. - - "reconnect", the client should close the connection and - reconnect. - If neither "reconnect" nor "redirect" is included in the instruction - list, the client can continue with this server after completing all - the instructions. - """ - args = self._readlist() - try: - self.ui, lui = _loadnewui(self.ui, args) - except error.ParseError as inst: - dispatch._formatparse(self.ui.warn, inst) - self.ui.flush() - self.cresult.write('exit 255') - return - newhash = hashstate.fromui(lui, self.hashstate.mtimepaths) - insts = [] - if newhash.mtimehash != self.hashstate.mtimehash: - addr = _hashaddress(self.baseaddress, self.hashstate.confighash) - insts.append('unlink %s' % addr) - # mtimehash is empty if one or more extensions fail to load. - # to be compatible with hg, still serve the client this time. - if self.hashstate.mtimehash: - insts.append('reconnect') - if newhash.confighash != self.hashstate.confighash: - addr = _hashaddress(self.baseaddress, newhash.confighash) - insts.append('redirect %s' % addr) - _log('validate: %s\n' % insts) - self.cresult.write('\0'.join(insts) or '\0') - - def chdir(self): - """Change current directory - - Note that the behavior of --cwd option is bit different from this. - It does not affect --config parameter. - """ - path = self._readstr() - if not path: - return - _log('chdir to %r\n' % path) - os.chdir(path) - - def setumask(self): - """Change umask""" - mask = struct.unpack('>I', self._read(4))[0] - _log('setumask %r\n' % mask) - os.umask(mask) - - def getpager(self): - """Read cmdargs and write pager command to r-channel if enabled - - If pager isn't enabled, this writes '\0' because channeledoutput - does not allow to write empty data. - """ - args = self._readlist() - try: - cmd, _func, args, options, _cmdoptions = dispatch._parse(self.ui, - args) - except (error.Abort, error.AmbiguousCommand, error.CommandError, - error.UnknownCommand): - cmd = None - options = {} - if not cmd or 'pager' not in options: - self.cresult.write('\0') - return - - pagercmd = _setuppagercmd(self.ui, options, cmd) - if pagercmd: - # Python's SIGPIPE is SIG_IGN by default. change to SIG_DFL so - # we can exit if the pipe to the pager is closed - if util.safehasattr(signal, 'SIGPIPE') and \ - signal.getsignal(signal.SIGPIPE) == signal.SIG_IGN: - signal.signal(signal.SIGPIPE, signal.SIG_DFL) - self.cresult.write(pagercmd) - else: - self.cresult.write('\0') - - def setenv(self): - """Clear and update os.environ - - Note that not all variables can make an effect on the running process. - """ - l = self._readlist() - try: - newenv = dict(s.split('=', 1) for s in l) - except ValueError: - raise ValueError('unexpected value in setenv request') - _log('setenv: %r\n' % sorted(newenv.keys())) - os.environ.clear() - os.environ.update(newenv) - - capabilities = commandserver.server.capabilities.copy() - capabilities.update({'attachio': attachio, - 'chdir': chdir, - 'getpager': getpager, - 'setenv': setenv, - 'setumask': setumask}) - -def _tempaddress(address): - return '%s.%d.tmp' % (address, os.getpid()) - -def _hashaddress(address, hashstr): - return '%s-%s' % (address, hashstr) - -class chgunixservicehandler(object): - """Set of operations for chg services""" - - pollinterval = 1 # [sec] - - def __init__(self, ui): - self.ui = ui - self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600) - self._lastactive = time.time() - - def bindsocket(self, sock, address): - self._inithashstate(address) - self._checkextensions() - self._bind(sock) - self._createsymlink() - - def _inithashstate(self, address): - self._baseaddress = address - if self.ui.configbool('chgserver', 'skiphash', False): - self._hashstate = None - self._realaddress = address - return - self._hashstate = hashstate.fromui(self.ui) - self._realaddress = _hashaddress(address, self._hashstate.confighash) - - def _checkextensions(self): - if not self._hashstate: - return - if extensions.notloaded(): - # one or more extensions failed to load. mtimehash becomes - # meaningless because we do not know the paths of those extensions. - # set mtimehash to an illegal hash value to invalidate the server. - self._hashstate.mtimehash = '' - - def _bind(self, sock): - # use a unique temp address so we can stat the file and do ownership - # check later - tempaddress = _tempaddress(self._realaddress) - util.bindunixsocket(sock, tempaddress) - self._socketstat = os.stat(tempaddress) - # rename will replace the old socket file if exists atomically. the - # old server will detect ownership change and exit. - util.rename(tempaddress, self._realaddress) - - def _createsymlink(self): - if self._baseaddress == self._realaddress: - return - tempaddress = _tempaddress(self._baseaddress) - os.symlink(os.path.basename(self._realaddress), tempaddress) - util.rename(tempaddress, self._baseaddress) - - def _issocketowner(self): - try: - stat = os.stat(self._realaddress) - return (stat.st_ino == self._socketstat.st_ino and - stat.st_mtime == self._socketstat.st_mtime) - except OSError: - return False - - def unlinksocket(self, address): - if not self._issocketowner(): - return - # it is possible to have a race condition here that we may - # remove another server's socket file. but that's okay - # since that server will detect and exit automatically and - # the client will start a new server on demand. - try: - os.unlink(self._realaddress) - except OSError as exc: - if exc.errno != errno.ENOENT: - raise - - def printbanner(self, address): - # no "listening at" message should be printed to simulate hg behavior - pass - - def shouldexit(self): - if not self._issocketowner(): - self.ui.debug('%s is not owned, exiting.\n' % self._realaddress) - return True - if time.time() - self._lastactive > self._idletimeout: - self.ui.debug('being idle too long. exiting.\n') - return True - return False - - def newconnection(self): - self._lastactive = time.time() - - def createcmdserver(self, repo, conn, fin, fout): - return chgcmdserver(self.ui, repo, fin, fout, conn, - self._hashstate, self._baseaddress) - -def chgunixservice(ui, repo, opts): - if repo: - # one chgserver can serve multiple repos. drop repo infomation - ui.setconfig('bundle', 'mainreporoot', '', 'repo') - h = chgunixservicehandler(ui) - return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h) - -def uisetup(ui): - commandserver._servicemap['chgunix'] = chgunixservice - - # CHGINTERNALMARK is temporarily set by chg client to detect if chg will - # start another chg. drop it to avoid possible side effects. - if 'CHGINTERNALMARK' in os.environ: - del os.environ['CHGINTERNALMARK'] diff -Nru mercurial-4.0.2~trusty1/hgext/color.py mercurial-4.1~rc~trusty1/hgext/color.py --- mercurial-4.0.2~trusty1/hgext/color.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/color.py 2017-01-18 16:54:51.000000000 +0000 @@ -164,14 +164,15 @@ from __future__ import absolute_import -import os - from mercurial.i18n import _ from mercurial import ( cmdutil, + color, commands, dispatch, + encoding, extensions, + pycompat, subrepo, ui as uimod, util, @@ -197,7 +198,6 @@ def _terminfosetup(ui, mode): '''Initialize terminfo data and the terminal if we're in terminfo mode.''' - global _terminfo_params # If we failed to load curses, we go ahead and return. if not _terminfo_params: return @@ -215,7 +215,7 @@ try: curses.setupterm() except curses.error as e: - _terminfo_params = {} + _terminfo_params.clear() return for key, (b, e, c) in _terminfo_params.items(): @@ -232,11 +232,9 @@ if mode == "terminfo": ui.warn(_("no terminfo entry for setab/setaf: reverting to " "ECMA-48 color\n")) - _terminfo_params = {} + _terminfo_params.clear() def _modesetup(ui, coloropt): - global _terminfo_params - if coloropt == 'debug': return 'debug' @@ -245,7 +243,8 @@ if not always and not auto: return None - formatted = always or (os.environ.get('TERM') != 'dumb' and ui.formatted()) + formatted = (always or (encoding.environ.get('TERM') != 'dumb' + and ui.formatted())) mode = ui.config('color', 'mode', 'auto') @@ -255,8 +254,8 @@ realmode = mode if mode == 'auto': - if os.name == 'nt': - term = os.environ.get('TERM') + if pycompat.osname == 'nt': + term = encoding.environ.get('TERM') # TERM won't be defined in a vanilla cmd.exe environment. # UNIX-like environments on Windows such as Cygwin and MSYS will @@ -275,18 +274,18 @@ def modewarn(): # only warn if color.mode was explicitly set and we're in - # an interactive terminal - if mode == realmode and ui.interactive(): + # a formatted terminal + if mode == realmode and ui.formatted(): ui.warn(_('warning: failed to set color mode to %s\n') % mode) if realmode == 'win32': - _terminfo_params = {} + _terminfo_params.clear() if not w32effects: modewarn() return None _effects.update(w32effects) elif realmode == 'ansi': - _terminfo_params = {} + _terminfo_params.clear() elif realmode == 'terminfo': _terminfosetup(ui, mode) if not _terminfo_params: @@ -325,61 +324,6 @@ except ImportError: _terminfo_params = {} -_styles = {'grep.match': 'red bold', - 'grep.linenumber': 'green', - 'grep.rev': 'green', - 'grep.change': 'green', - 'grep.sep': 'cyan', - 'grep.filename': 'magenta', - 'grep.user': 'magenta', - 'grep.date': 'magenta', - 'bookmarks.active': 'green', - 'branches.active': 'none', - 'branches.closed': 'black bold', - 'branches.current': 'green', - 'branches.inactive': 'none', - 'diff.changed': 'white', - 'diff.deleted': 'red', - 'diff.diffline': 'bold', - 'diff.extended': 'cyan bold', - 'diff.file_a': 'red bold', - 'diff.file_b': 'green bold', - 'diff.hunk': 'magenta', - 'diff.inserted': 'green', - 'diff.tab': '', - 'diff.trailingwhitespace': 'bold red_background', - 'changeset.public' : '', - 'changeset.draft' : '', - 'changeset.secret' : '', - 'diffstat.deleted': 'red', - 'diffstat.inserted': 'green', - 'histedit.remaining': 'red bold', - 'ui.prompt': 'yellow', - 'log.changeset': 'yellow', - 'patchbomb.finalsummary': '', - 'patchbomb.from': 'magenta', - 'patchbomb.to': 'cyan', - 'patchbomb.subject': 'green', - 'patchbomb.diffstats': '', - 'rebase.rebased': 'blue', - 'rebase.remaining': 'red bold', - 'resolve.resolved': 'green bold', - 'resolve.unresolved': 'red bold', - 'shelve.age': 'cyan', - 'shelve.newest': 'green bold', - 'shelve.name': 'blue bold', - 'status.added': 'green bold', - 'status.clean': 'none', - 'status.copied': 'none', - 'status.deleted': 'cyan bold underline', - 'status.ignored': 'black bold', - 'status.modified': 'blue bold', - 'status.removed': 'red bold', - 'status.unknown': 'magenta bold underline', - 'tags.normal': 'green', - 'tags.local': 'black bold'} - - def _effect_str(effect): '''Helper function for render_effects().''' @@ -415,10 +359,6 @@ stop = _effect_str('none') return ''.join([start, text, stop]) -def extstyles(): - for name, ext in extensions.extensions(): - _styles.update(getattr(ext, 'colortable', {})) - def valideffect(effect): 'Determine if the effect is valid or not.' good = False @@ -442,7 +382,7 @@ ui.warn(_("ignoring unknown color/effect %r " "(configured in color.%s)\n") % (e, status)) - _styles[status] = ' '.join(good) + color._styles[status] = ' '.join(good) class colorui(uimod.ui): _colormode = 'ansi' @@ -495,15 +435,15 @@ effects = [] for l in label.split(): - s = _styles.get(l, '') + s = color._styles.get(l, '') if s: effects.append(s) elif valideffect(l): effects.append(l) effects = ' '.join(effects) if effects: - return '\n'.join([render_effects(s, effects) - for s in msg.split('\n')]) + return '\n'.join([render_effects(line, effects) + for line in msg.split('\n')]) return msg def uisetup(ui): @@ -516,7 +456,6 @@ mode = _modesetup(ui_, opts['color']) colorui._colormode = mode if mode and mode != 'debug': - extstyles() configstyles(ui_) return orig(ui_, opts, cmd, cmdfunc) def colorgit(orig, gitsub, commands, env=None, stream=False, cwd=None): @@ -536,24 +475,52 @@ _("when to colorize (boolean, always, auto, never, or debug)"), _('TYPE'))) -@command('debugcolor', [], 'hg debugcolor') +@command('debugcolor', + [('', 'style', None, _('show all configured styles'))], + 'hg debugcolor') def debugcolor(ui, repo, **opts): - global _styles - _styles = {} - for effect in _effects.keys(): - _styles[effect] = effect - if _terminfo_params: - for k, v in ui.configitems('color'): - if k.startswith('color.'): - _styles[k] = k[6:] - elif k.startswith('terminfo.'): - _styles[k] = k[9:] + """show available color, effects or style""" ui.write(('color mode: %s\n') % ui._colormode) - ui.write(_('available colors:\n')) - for colorname, label in _styles.items(): - ui.write(('%s\n') % colorname, label=label) + if opts.get('style'): + return _debugdisplaystyle(ui) + else: + return _debugdisplaycolor(ui) + +def _debugdisplaycolor(ui): + oldstyle = color._styles.copy() + try: + color._styles.clear() + for effect in _effects.keys(): + color._styles[effect] = effect + if _terminfo_params: + for k, v in ui.configitems('color'): + if k.startswith('color.'): + color._styles[k] = k[6:] + elif k.startswith('terminfo.'): + color._styles[k] = k[9:] + ui.write(_('available colors:\n')) + # sort label with a '_' after the other to group '_background' entry. + items = sorted(color._styles.items(), + key=lambda i: ('_' in i[0], i[0], i[1])) + for colorname, label in items: + ui.write(('%s\n') % colorname, label=label) + finally: + color._styles.clear() + color._styles.update(oldstyle) + +def _debugdisplaystyle(ui): + ui.write(_('available style:\n')) + width = max(len(s) for s in color._styles) + for label, effects in sorted(color._styles.items()): + ui.write('%s' % label, label=label) + if effects: + # 50 + ui.write(': ') + ui.write(' ' * (max(0, width - len(label)))) + ui.write(', '.join(ui.label(e, e) for e in effects.split())) + ui.write('\n') -if os.name != 'nt': +if pycompat.osname != 'nt': w32effects = None else: import ctypes @@ -661,7 +628,7 @@ # determine console attributes based on labels for l in label.split(): - style = _styles.get(l, '') + style = color._styles.get(l, '') for effect in style.split(): try: attr = mapcolor(w32effects[effect], attr) diff -Nru mercurial-4.0.2~trusty1/hgext/convert/common.py mercurial-4.1~rc~trusty1/hgext/convert/common.py --- mercurial-4.0.2~trusty1/hgext/convert/common.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/common.py 2017-01-18 16:54:51.000000000 +0000 @@ -55,7 +55,7 @@ class commit(object): def __init__(self, author, date, desc, parents, branch=None, rev=None, - extra={}, sortkey=None, saverev=True, phase=phases.draft, + extra=None, sortkey=None, saverev=True, phase=phases.draft, optparents=None): self.author = author or 'unknown' self.date = date or '0 0' @@ -64,7 +64,7 @@ self.optparents = optparents or [] # will be used if already converted self.branch = branch self.rev = rev - self.extra = extra + self.extra = extra or {} self.sortkey = sortkey self.saverev = saverev self.phase = phase @@ -454,7 +454,7 @@ if err.errno != errno.ENOENT: raise return - for i, line in enumerate(fp): + for i, line in enumerate(util.iterfile(fp)): line = line.splitlines()[0].rstrip() if not line: # Ignore blank lines diff -Nru mercurial-4.0.2~trusty1/hgext/convert/convcmd.py mercurial-4.1~rc~trusty1/hgext/convert/convcmd.py --- mercurial-4.0.2~trusty1/hgext/convert/convcmd.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/convcmd.py 2017-01-18 16:54:51.000000000 +0000 @@ -201,7 +201,7 @@ m = {} try: fp = open(path, 'r') - for i, line in enumerate(fp): + for i, line in enumerate(util.iterfile(fp)): line = line.splitlines()[0].rstrip() if not line: # Ignore blank lines diff -Nru mercurial-4.0.2~trusty1/hgext/convert/cvsps.py mercurial-4.1~rc~trusty1/hgext/convert/cvsps.py --- mercurial-4.0.2~trusty1/hgext/convert/cvsps.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/cvsps.py 2017-01-18 16:54:51.000000000 +0000 @@ -11,7 +11,9 @@ from mercurial.i18n import _ from mercurial import ( + encoding, hook, + pycompat, util, ) @@ -136,8 +138,8 @@ except IOError: raise logerror(_('not a CVS sandbox')) - if prefix and not prefix.endswith(os.sep): - prefix += os.sep + if prefix and not prefix.endswith(pycompat.ossep): + prefix += pycompat.ossep # Use the Root file in the sandbox, if it exists try: @@ -146,7 +148,7 @@ pass if not root: - root = os.environ.get('CVSROOT', '') + root = encoding.environ.get('CVSROOT', '') # read log cache if one exists oldlog = [] diff -Nru mercurial-4.0.2~trusty1/hgext/convert/cvs.py mercurial-4.1~rc~trusty1/hgext/convert/cvs.py --- mercurial-4.0.2~trusty1/hgext/convert/cvs.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/cvs.py 2017-01-18 16:54:51.000000000 +0000 @@ -15,6 +15,7 @@ from mercurial import ( encoding, error, + pycompat, util, ) @@ -69,7 +70,7 @@ raise error.Abort(_('revision %s is not a patchset number') % self.revs[0]) - d = os.getcwd() + d = pycompat.getcwd() try: os.chdir(self.path) id = None @@ -188,7 +189,7 @@ if conntype != "pserver": if conntype == "rsh": - rsh = os.environ.get("CVS_RSH") or "ssh" + rsh = encoding.environ.get("CVS_RSH") or "ssh" if user: cmd = [rsh, '-l', user, host] + cmd else: diff -Nru mercurial-4.0.2~trusty1/hgext/convert/git.py mercurial-4.1~rc~trusty1/hgext/convert/git.py --- mercurial-4.0.2~trusty1/hgext/convert/git.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/git.py 2017-01-18 16:54:51.000000000 +0000 @@ -31,6 +31,18 @@ def hgsubstate(self): return "%s %s" % (self.node, self.path) +# Keys in extra fields that should not be copied if the user requests. +bannedextrakeys = set([ + # Git commit object built-ins. + 'tree', + 'parent', + 'author', + 'committer', + # Mercurial built-ins. + 'branch', + 'close', +]) + class convert_git(common.converter_source, common.commandline): # Windows does not support GIT_DIR= construct while other systems # cannot remove environment variable. Just assume none have @@ -78,6 +90,10 @@ False) if findcopiesharder: self.simopt.append('--find-copies-harder') + + renamelimit = ui.configint('convert', 'git.renamelimit', + default=400) + self.simopt.append('-l%d' % renamelimit) else: self.simopt = [] @@ -88,6 +104,54 @@ self.catfilepipe = self.gitpipe('cat-file', '--batch') + self.copyextrakeys = self.ui.configlist('convert', 'git.extrakeys') + banned = set(self.copyextrakeys) & bannedextrakeys + if banned: + raise error.Abort(_('copying of extra key is forbidden: %s') % + _(', ').join(sorted(banned))) + + committeractions = self.ui.configlist('convert', 'git.committeractions', + 'messagedifferent') + + messagedifferent = None + messagealways = None + for a in committeractions: + if a.startswith(('messagedifferent', 'messagealways')): + k = a + v = None + if '=' in a: + k, v = a.split('=', 1) + + if k == 'messagedifferent': + messagedifferent = v or 'committer:' + elif k == 'messagealways': + messagealways = v or 'committer:' + + if messagedifferent and messagealways: + raise error.Abort(_('committeractions cannot define both ' + 'messagedifferent and messagealways')) + + dropcommitter = 'dropcommitter' in committeractions + replaceauthor = 'replaceauthor' in committeractions + + if dropcommitter and replaceauthor: + raise error.Abort(_('committeractions cannot define both ' + 'dropcommitter and replaceauthor')) + + if dropcommitter and messagealways: + raise error.Abort(_('committeractions cannot define both ' + 'dropcommitter and messagealways')) + + if not messagedifferent and not messagealways: + messagedifferent = 'committer:' + + self.committeractions = { + 'dropcommitter': dropcommitter, + 'replaceauthor': replaceauthor, + 'messagedifferent': messagedifferent, + 'messagealways': messagealways, + } + def after(self): for f in self.catfilepipe: f.close() @@ -275,6 +339,7 @@ l = c[:end].splitlines() parents = [] author = committer = None + extra = {} for e in l[1:]: n, v = e.split(" ", 1) if n == "author": @@ -291,16 +356,32 @@ committer = self.recode(committer) if n == "parent": parents.append(v) + if n in self.copyextrakeys: + extra[n] = v + + if self.committeractions['dropcommitter']: + committer = None + elif self.committeractions['replaceauthor']: + author = committer + + if committer: + messagealways = self.committeractions['messagealways'] + messagedifferent = self.committeractions['messagedifferent'] + if messagealways: + message += '\n%s %s\n' % (messagealways, committer) + elif messagedifferent and author != committer: + message += '\n%s %s\n' % (messagedifferent, committer) - if committer and committer != author: - message += "\ncommitter: %s\n" % committer tzs, tzh, tzm = tz[-5:-4] + "1", tz[-4:-2], tz[-2:] tz = -int(tzs) * (int(tzh) * 3600 + int(tzm)) date = tm + " " + str(tz) + saverev = self.ui.configbool('convert', 'git.saverev', True) c = common.commit(parents=parents, date=date, author=author, desc=message, - rev=version) + rev=version, + extra=extra, + saverev=saverev) return c def numcommits(self): diff -Nru mercurial-4.0.2~trusty1/hgext/convert/hg.py mercurial-4.1~rc~trusty1/hgext/convert/hg.py --- mercurial-4.0.2~trusty1/hgext/convert/hg.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/hg.py 2017-01-18 16:54:51.000000000 +0000 @@ -352,7 +352,7 @@ p2 = node if self.filemapmode and nparents == 1: - man = self.repo.manifest + man = self.repo.manifestlog._revlog mnode = self.repo.changelog.read(nodemod.bin(p2))[0] closed = 'close' in commit.extra if not closed and not man.cmp(m1node, man.revision(mnode)): diff -Nru mercurial-4.0.2~trusty1/hgext/convert/__init__.py mercurial-4.1~rc~trusty1/hgext/convert/__init__.py --- mercurial-4.0.2~trusty1/hgext/convert/__init__.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/__init__.py 2017-01-18 16:54:51.000000000 +0000 @@ -320,10 +320,56 @@ is very expensive for large projects, and is only effective when ``convert.git.similarity`` is greater than 0. The default is False. + :convert.git.renamelimit: perform rename and copy detection up to this + many changed files in a commit. Increasing this will make rename + and copy detection more accurate but will significantly slow down + computation on large projects. The option is only relevant if + ``convert.git.similarity`` is greater than 0. The default is + ``400``. + + :convert.git.committeractions: list of actions to take when processing + author and committer values. + + Git commits have separate author (who wrote the commit) and committer + (who applied the commit) fields. Not all destinations support separate + author and committer fields (including Mercurial). This config option + controls what to do with these author and committer fields during + conversion. + + A value of ``messagedifferent`` will append a ``committer: ...`` + line to the commit message if the Git committer is different from the + author. The prefix of that line can be specified using the syntax + ``messagedifferent=``. e.g. ``messagedifferent=git-committer:``. + When a prefix is specified, a space will always be inserted between the + prefix and the value. + + ``messagealways`` behaves like ``messagedifferent`` except it will + always result in a ``committer: ...`` line being appended to the commit + message. This value is mutually exclusive with ``messagedifferent``. + + ``dropcommitter`` will remove references to the committer. Only + references to the author will remain. Actions that add references + to the committer will have no effect when this is set. + + ``replaceauthor`` will replace the value of the author field with + the committer. Other actions that add references to the committer + will still take effect when this is set. + + The default is ``messagedifferent``. + + :convert.git.extrakeys: list of extra keys from commit metadata to copy to + the destination. Some Git repositories store extra metadata in commits. + By default, this non-default metadata will be lost during conversion. + Setting this config option can retain that metadata. Some built-in + keys such as ``parent`` and ``branch`` are not allowed to be copied. + :convert.git.remoteprefix: remote refs are converted as bookmarks with ``convert.git.remoteprefix`` as a prefix followed by a /. The default is 'remote'. + :convert.git.saverev: whether to store the original Git commit ID in the + metadata of the destination commit. The default is True. + :convert.git.skipsubmodules: does not convert root level .gitmodules files or files with 160000 mode indicating a submodule. Default is False. diff -Nru mercurial-4.0.2~trusty1/hgext/convert/p4.py mercurial-4.1~rc~trusty1/hgext/convert/p4.py --- mercurial-4.0.2~trusty1/hgext/convert/p4.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/p4.py 2017-01-18 16:54:51.000000000 +0000 @@ -55,18 +55,9 @@ common.checktool('p4', abort=False) - self.p4changes = {} - self.heads = {} - self.changeset = {} - self.files = {} - self.copies = {} - self.tags = {} - self.lastbranch = {} - self.parent = {} + self.revmap = {} self.encoding = self.ui.config('convert', 'p4.encoding', default=convcmd.orig_encoding) - self.depotname = {} # mapping from local name to depot name - self.localname = {} # mapping from depot name to local name self.re_type = re.compile( "([a-z]+)?(text|binary|symlink|apple|resource|unicode|utf\d+)" "(\+\w+)?$") @@ -78,24 +69,46 @@ if revs and len(revs) > 1: raise error.Abort(_("p4 source does not support specifying " "multiple revisions")) - self._parse(ui, path) + + def setrevmap(self, revmap): + """Sets the parsed revmap dictionary. + + Revmap stores mappings from a source revision to a target revision. + It is set in convertcmd.convert and provided by the user as a file + on the commandline. + + Revisions in the map are considered beeing present in the + repository and ignored during _parse(). This allows for incremental + imports if a revmap is provided. + """ + self.revmap = revmap def _parse_view(self, path): "Read changes affecting the path" cmd = 'p4 -G changes -s submitted %s' % util.shellquote(path) stdout = util.popen(cmd, mode='rb') + p4changes = {} for d in loaditer(stdout): c = d.get("change", None) if c: - self.p4changes[c] = True + p4changes[c] = True + return p4changes def _parse(self, ui, path): "Prepare list of P4 filenames and revisions to import" + p4changes = {} + changeset = {} + files_map = {} + copies_map = {} + localname = {} + depotname = {} + heads = [] + ui.status(_('reading p4 views\n')) # read client spec or view if "/" in path: - self._parse_view(path) + p4changes.update(self._parse_view(path)) if path.startswith("//") and path.endswith("/..."): views = {path[:-3]:""} else: @@ -108,7 +121,7 @@ for client in clientspec: if client.startswith("View"): sview, cview = clientspec[client].split() - self._parse_view(sview) + p4changes.update(self._parse_view(sview)) if sview.endswith("...") and cview.endswith("..."): sview = sview[:-3] cview = cview[:-3] @@ -117,8 +130,8 @@ views[sview] = cview # list of changes that affect our source files - self.p4changes = self.p4changes.keys() - self.p4changes.sort(key=int) + p4changes = p4changes.keys() + p4changes.sort(key=int) # list with depot pathnames, longest first vieworder = views.keys() @@ -126,32 +139,31 @@ # handle revision limiting startrev = self.ui.config('convert', 'p4.startrev', default=0) - self.p4changes = [x for x in self.p4changes - if ((not startrev or int(x) >= int(startrev)) and - (not self.revs or int(x) <= int(self.revs[0])))] # now read the full changelists to get the list of file revisions ui.status(_('collecting p4 changelists\n')) lastid = None - for change in self.p4changes: - cmd = "p4 -G describe -s %s" % change - stdout = util.popen(cmd, mode='rb') - d = marshal.load(stdout) - desc = self.recode(d.get("desc", "")) - shortdesc = desc.split("\n", 1)[0] - t = '%s %s' % (d["change"], repr(shortdesc)[1:-1]) - ui.status(util.ellipsis(t, 80) + '\n') + for change in p4changes: + if startrev and int(change) < int(startrev): + continue + if self.revs and int(change) > int(self.revs[0]): + continue + if change in self.revmap: + # Ignore already present revisions, but set the parent pointer. + lastid = change + continue if lastid: parents = [lastid] else: parents = [] - date = (int(d["time"]), 0) # timezone not set - c = common.commit(author=self.recode(d["user"]), - date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'), - parents=parents, desc=desc, branch=None, - extra={"p4": change}) + d = self._fetch_revision(change) + c = self._construct_commit(d, parents) + + shortdesc = c.desc.splitlines(True)[0].rstrip('\r\n') + t = '%s %s' % (c.rev, repr(shortdesc)[1:-1]) + ui.status(util.ellipsis(t, 80) + '\n') files = [] copies = {} @@ -166,15 +178,15 @@ break if filename: files.append((filename, d["rev%d" % i])) - self.depotname[filename] = oldname + depotname[filename] = oldname if (d.get("action%d" % i) == "move/add"): copiedfiles.append(filename) - self.localname[oldname] = filename + localname[oldname] = filename i += 1 # Collect information about copied files for filename in copiedfiles: - oldname = self.depotname[filename] + oldname = depotname[filename] flcmd = 'p4 -G filelog %s' \ % util.shellquote(oldname) @@ -196,8 +208,8 @@ j += 1 i += 1 - if copiedoldname and copiedoldname in self.localname: - copiedfilename = self.localname[copiedoldname] + if copiedoldname and copiedoldname in localname: + copiedfilename = localname[copiedoldname] break if copiedfilename: @@ -206,13 +218,45 @@ ui.warn(_("cannot find source for copied file: %s@%s\n") % (filename, change)) - self.changeset[change] = c - self.files[change] = files - self.copies[change] = copies + changeset[change] = c + files_map[change] = files + copies_map[change] = copies lastid = change - if lastid: - self.heads = [lastid] + if lastid and len(changeset) > 0: + heads = [lastid] + + return { + 'changeset': changeset, + 'files': files_map, + 'copies': copies_map, + 'heads': heads, + 'depotname': depotname, + } + + @util.propertycache + def _parse_once(self): + return self._parse(self.ui, self.path) + + @util.propertycache + def copies(self): + return self._parse_once['copies'] + + @util.propertycache + def files(self): + return self._parse_once['files'] + + @util.propertycache + def changeset(self): + return self._parse_once['changeset'] + + @util.propertycache + def heads(self): + return self._parse_once['heads'] + + @util.propertycache + def depotname(self): + return self._parse_once['depotname'] def getheads(self): return self.heads @@ -286,11 +330,39 @@ raise error.Abort(_("convert from p4 does not support --full")) return self.files[rev], self.copies[rev], set() + def _construct_commit(self, obj, parents=None): + """ + Constructs a common.commit object from an unmarshalled + `p4 describe` output + """ + desc = self.recode(obj.get("desc", "")) + date = (int(obj["time"]), 0) # timezone not set + if parents is None: + parents = [] + + return common.commit(author=self.recode(obj["user"]), + date=util.datestr(date, '%Y-%m-%d %H:%M:%S %1%2'), + parents=parents, desc=desc, branch=None, rev=obj['change'], + extra={"p4": obj['change'], "convert_revision": obj['change']}) + + def _fetch_revision(self, rev): + """Return an output of `p4 describe` including author, commit date as + a dictionary.""" + cmd = "p4 -G describe -s %s" % rev + stdout = util.popen(cmd, mode='rb') + return marshal.load(stdout) + def getcommit(self, rev): - return self.changeset[rev] + if rev in self.changeset: + return self.changeset[rev] + elif rev in self.revmap: + d = self._fetch_revision(rev) + return self._construct_commit(d, parents=None) + raise error.Abort( + _("cannot find %s in the revmap or parsed changesets") % rev) def gettags(self): - return self.tags + return {} def getchangedfiles(self, rev, i): return sorted([x[0] for x in self.files[rev]]) diff -Nru mercurial-4.0.2~trusty1/hgext/convert/subversion.py mercurial-4.1~rc~trusty1/hgext/convert/subversion.py --- mercurial-4.0.2~trusty1/hgext/convert/subversion.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/convert/subversion.py 2017-01-18 16:54:51.000000000 +0000 @@ -5,7 +5,6 @@ import os import re -import sys import tempfile import xml.dom.minidom @@ -13,8 +12,8 @@ from mercurial import ( encoding, error, + pycompat, scmutil, - strutil, util, ) @@ -104,7 +103,7 @@ pass if os.path.isdir(path): path = os.path.normpath(os.path.abspath(path)) - if os.name == 'nt': + if pycompat.osname == 'nt': path = '/' + util.normpath(path) # Module URL is later compared with the repository URL returned # by svn API, which is UTF-8. @@ -164,10 +163,8 @@ raise error.Abort(_('debugsvnlog could not load Subversion python ' 'bindings')) - util.setbinary(sys.stdin) - util.setbinary(sys.stdout) - args = decodeargs(sys.stdin.read()) - get_log_child(sys.stdout, *args) + args = decodeargs(ui.fin.read()) + get_log_child(ui.fout, *args) class logstream(object): """Interruptible revision log iterator.""" @@ -257,8 +254,8 @@ try: proto, path = url.split('://', 1) if proto == 'file': - if (os.name == 'nt' and path[:1] == '/' and path[1:2].isalpha() - and path[2:6].lower() == '%3a/'): + if (pycompat.osname == 'nt' and path[:1] == '/' + and path[1:2].isalpha() and path[2:6].lower() == '%3a/'): path = path[:2] + ':/' + path[6:] path = urlreq.url2pathname(path) except ValueError: @@ -1122,7 +1119,7 @@ self.delexec = [] self.copies = [] self.wc = None - self.cwd = os.getcwd() + self.cwd = pycompat.getcwd() created = False if os.path.isfile(os.path.join(path, '.svn', 'entries')): @@ -1142,7 +1139,8 @@ path = '/' + path path = 'file://' + path - wcpath = os.path.join(os.getcwd(), os.path.basename(path) + '-wc') + wcpath = os.path.join(pycompat.getcwd(), os.path.basename(path) + + '-wc') ui.status(_('initializing svn working copy %r\n') % os.path.basename(wcpath)) self.run0('checkout', path, wcpath) @@ -1240,7 +1238,8 @@ for f in files: if os.path.isdir(self.wjoin(f)): dirs.add(f) - for i in strutil.rfindall(f, '/'): + i = len(f) + for i in iter(lambda: f.rfind('/', 0, i), -1): dirs.add(f[:i]) return dirs diff -Nru mercurial-4.0.2~trusty1/hgext/extdiff.py mercurial-4.1~rc~trusty1/hgext/extdiff.py --- mercurial-4.0.2~trusty1/hgext/extdiff.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/extdiff.py 2017-01-18 16:54:51.000000000 +0000 @@ -64,7 +64,6 @@ import os import re -import shlex import shutil import tempfile from mercurial.i18n import _ @@ -78,6 +77,7 @@ commands, error, filemerge, + pycompat, scmutil, util, ) @@ -371,7 +371,7 @@ if path: # case "cmd = path opts" cmdline = path - diffopts = len(shlex.split(cmdline)) > 1 + diffopts = len(pycompat.shlexsplit(cmdline)) > 1 else: # case "cmd =" path = util.findexe(cmd) diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/__init__.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/__init__.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/__init__.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/__init__.py 2017-01-18 16:54:51.000000000 +0000 @@ -94,15 +94,16 @@ import hashlib import os import stat -import sys from mercurial.i18n import _ from mercurial import ( context, + encoding, extensions, localrepo, merge, pathutil, + pycompat, scmutil, util, ) @@ -292,7 +293,7 @@ if normalize: foldmap = dict((normcase(k), k) for k in results) - switch_slashes = os.sep == '\\' + switch_slashes = pycompat.ossep == '\\' # The order of the results is, strictly speaking, undefined. # For case changes on a case insensitive filesystem we may receive # two entries, one with exists=True and another with exists=False. @@ -392,8 +393,8 @@ def _cmpsets(l1, l2): try: - if 'FSMONITOR_LOG_FILE' in os.environ: - fn = os.environ['FSMONITOR_LOG_FILE'] + if 'FSMONITOR_LOG_FILE' in encoding.environ: + fn = encoding.environ['FSMONITOR_LOG_FILE'] f = open(fn, 'wb') else: fn = 'fsmonitorfail.log' @@ -434,7 +435,7 @@ updatestate = (parentworking and match.always() and not isinstance(ctx2, (context.workingcommitctx, context.memctx)) and - 'HG_PENDING' not in os.environ) + 'HG_PENDING' not in encoding.environ) try: if self._fsmonitorstate.walk_on_invalidate: @@ -545,7 +546,7 @@ def extsetup(ui): wrapfilecache(localrepo.localrepository, 'dirstate', wrapdirstate) - if sys.platform == 'darwin': + if pycompat.sysplatform == 'darwin': # An assist for avoiding the dangling-symlink fsevents bug extensions.wrapfunction(os, 'symlink', wrapsymlink) @@ -563,7 +564,7 @@ pass class state_update(object): - ''' This context mananger is responsible for dispatching the state-enter + ''' This context manager is responsible for dispatching the state-enter and state-leave signals to the watchman service ''' def __init__(self, repo, node, distance, partial): diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/bser.c mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/bser.c --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/bser.c 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/bser.c 2017-01-18 16:54:51.000000000 +0000 @@ -29,11 +29,27 @@ */ #include +#include #ifdef _MSC_VER #define inline __inline -#include "msc_stdint.h" +#if _MSC_VER >= 1800 +#include +#else +// The compiler associated with Python 2.7 on Windows doesn't ship +// with stdint.h, so define the small subset that we use here. +typedef __int8 int8_t; +typedef __int16 int16_t; +typedef __int32 int32_t; +typedef __int64 int64_t; +typedef unsigned __int8 uint8_t; +typedef unsigned __int16 uint16_t; +typedef unsigned __int32 uint32_t; +typedef unsigned __int64 uint64_t; +#define UINT32_MAX 4294967295U +#endif #endif +// clang-format off /* Return the smallest size int that can store the value */ #define INT_SIZE(x) (((x) == ((int8_t)x)) ? 1 : \ ((x) == ((int16_t)x)) ? 2 : \ @@ -41,7 +57,7 @@ #define BSER_ARRAY 0x00 #define BSER_OBJECT 0x01 -#define BSER_STRING 0x02 +#define BSER_BYTESTRING 0x02 #define BSER_INT8 0x03 #define BSER_INT16 0x04 #define BSER_INT32 0x05 @@ -52,6 +68,8 @@ #define BSER_NULL 0x0a #define BSER_TEMPLATE 0x0b #define BSER_SKIP 0x0c +#define BSER_UTF8STRING 0x0d +// clang-format on // An immutable object representation of BSER_OBJECT. // Rather than build a hash table, key -> value are obtained @@ -64,24 +82,27 @@ // approach, this is still faster for the mercurial use case // as it helps to eliminate creating N other objects to // represent the stat information in the hgwatchman extension +// clang-format off typedef struct { PyObject_HEAD PyObject *keys; // tuple of field names PyObject *values; // tuple of values } bserObject; +// clang-format on -static Py_ssize_t bserobj_tuple_length(PyObject *o) { - bserObject *obj = (bserObject*)o; +static Py_ssize_t bserobj_tuple_length(PyObject* o) { + bserObject* obj = (bserObject*)o; return PySequence_Length(obj->keys); } -static PyObject *bserobj_tuple_item(PyObject *o, Py_ssize_t i) { - bserObject *obj = (bserObject*)o; +static PyObject* bserobj_tuple_item(PyObject* o, Py_ssize_t i) { + bserObject* obj = (bserObject*)o; return PySequence_GetItem(obj->values, i); } +// clang-format off static PySequenceMethods bserobj_sq = { bserobj_tuple_length, /* sq_length */ 0, /* sq_concat */ @@ -92,49 +113,72 @@ 0, /* sq_inplace_concat */ 0 /* sq_inplace_repeat */ }; +// clang-format on -static void bserobj_dealloc(PyObject *o) { - bserObject *obj = (bserObject*)o; +static void bserobj_dealloc(PyObject* o) { + bserObject* obj = (bserObject*)o; Py_CLEAR(obj->keys); Py_CLEAR(obj->values); PyObject_Del(o); } -static PyObject *bserobj_getattrro(PyObject *o, PyObject *name) { - bserObject *obj = (bserObject*)o; +static PyObject* bserobj_getattrro(PyObject* o, PyObject* name) { + bserObject* obj = (bserObject*)o; Py_ssize_t i, n; - const char *namestr; + PyObject* name_bytes = NULL; + PyObject* ret = NULL; + const char* namestr; if (PyIndex_Check(name)) { i = PyNumber_AsSsize_t(name, PyExc_IndexError); if (i == -1 && PyErr_Occurred()) { - return NULL; + goto bail; } - return PySequence_GetItem(obj->values, i); + ret = PySequence_GetItem(obj->values, i); + goto bail; } + // We can be passed in Unicode objects here -- we don't support anything other + // than UTF-8 for keys. + if (PyUnicode_Check(name)) { + name_bytes = PyUnicode_AsUTF8String(name); + if (name_bytes == NULL) { + goto bail; + } + namestr = PyBytes_AsString(name_bytes); + } else { + namestr = PyBytes_AsString(name); + } + + if (namestr == NULL) { + goto bail; + } // hack^Wfeature to allow mercurial to use "st_size" to reference "size" - namestr = PyString_AsString(name); if (!strncmp(namestr, "st_", 3)) { namestr += 3; } n = PyTuple_GET_SIZE(obj->keys); for (i = 0; i < n; i++) { - const char *item_name = NULL; - PyObject *key = PyTuple_GET_ITEM(obj->keys, i); + const char* item_name = NULL; + PyObject* key = PyTuple_GET_ITEM(obj->keys, i); - item_name = PyString_AsString(key); + item_name = PyBytes_AsString(key); if (!strcmp(item_name, namestr)) { - return PySequence_GetItem(obj->values, i); + ret = PySequence_GetItem(obj->values, i); + goto bail; } } - PyErr_Format(PyExc_AttributeError, - "bserobject has no attribute '%.400s'", namestr); - return NULL; + + PyErr_Format( + PyExc_AttributeError, "bserobject has no attribute '%.400s'", namestr); +bail: + Py_XDECREF(name_bytes); + return ret; } +// clang-format off static PyMappingMethods bserobj_map = { bserobj_tuple_length, /* mp_length */ bserobj_getattrro, /* mp_subscript */ @@ -181,20 +225,27 @@ 0, /* tp_alloc */ 0, /* tp_new */ }; +// clang-format on +typedef struct loads_ctx { + int mutable; + const char* value_encoding; + const char* value_errors; + uint32_t bser_version; + uint32_t bser_capabilities; +} unser_ctx_t; -static PyObject *bser_loads_recursive(const char **ptr, const char *end, - int mutable); +static PyObject* +bser_loads_recursive(const char** ptr, const char* end, const unser_ctx_t* ctx); static const char bser_true = BSER_TRUE; static const char bser_false = BSER_FALSE; static const char bser_null = BSER_NULL; -static const char bser_string_hdr = BSER_STRING; +static const char bser_bytestring_hdr = BSER_BYTESTRING; static const char bser_array_hdr = BSER_ARRAY; static const char bser_object_hdr = BSER_OBJECT; -static inline uint32_t next_power_2(uint32_t n) -{ +static inline uint32_t next_power_2(uint32_t n) { n |= (n >> 16); n |= (n >> 8); n |= (n >> 4); @@ -205,16 +256,17 @@ // A buffer we use for building up the serialized result struct bser_buffer { - char *buf; + char* buf; int wpos, allocd; + uint32_t bser_version; + uint32_t capabilities; }; typedef struct bser_buffer bser_t; -static int bser_append(bser_t *bser, const char *data, uint32_t len) -{ +static int bser_append(bser_t* bser, const char* data, uint32_t len) { int newlen = next_power_2(bser->wpos + len); if (newlen > bser->allocd) { - char *nbuf = realloc(bser->buf, newlen); + char* nbuf = realloc(bser->buf, newlen); if (!nbuf) { return 0; } @@ -228,40 +280,46 @@ return 1; } -static int bser_init(bser_t *bser) -{ +static int bser_init(bser_t* bser, uint32_t version, uint32_t capabilities) { bser->allocd = 8192; bser->wpos = 0; bser->buf = malloc(bser->allocd); - + bser->bser_version = version; + bser->capabilities = capabilities; if (!bser->buf) { return 0; } - // Leave room for the serialization header, which includes - // our overall length. To make things simpler, we'll use an - // int32 for the header +// Leave room for the serialization header, which includes +// our overall length. To make things simpler, we'll use an +// int32 for the header #define EMPTY_HEADER "\x00\x01\x05\x00\x00\x00\x00" - bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER)-1); + +// Version 2 also carries an integer indicating the capabilities. The +// capabilities integer comes before the PDU size. +#define EMPTY_HEADER_V2 "\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00" + if (version == 2) { + bser_append(bser, EMPTY_HEADER_V2, sizeof(EMPTY_HEADER_V2) - 1); + } else { + bser_append(bser, EMPTY_HEADER, sizeof(EMPTY_HEADER) - 1); + } return 1; } -static void bser_dtor(bser_t *bser) -{ +static void bser_dtor(bser_t* bser) { free(bser->buf); bser->buf = NULL; } -static int bser_long(bser_t *bser, int64_t val) -{ +static int bser_long(bser_t* bser, int64_t val) { int8_t i8; int16_t i16; int32_t i32; int64_t i64; char sz; int size = INT_SIZE(val); - char *iptr; + char* iptr; switch (size) { case 1: @@ -285,8 +343,7 @@ iptr = (char*)&i64; break; default: - PyErr_SetString(PyExc_RuntimeError, - "Cannot represent this long value!?"); + PyErr_SetString(PyExc_RuntimeError, "Cannot represent this long value!?"); return 0; } @@ -297,25 +354,24 @@ return bser_append(bser, iptr, size); } -static int bser_string(bser_t *bser, PyObject *sval) -{ - char *buf = NULL; +static int bser_bytestring(bser_t* bser, PyObject* sval) { + char* buf = NULL; Py_ssize_t len; int res; - PyObject *utf = NULL; + PyObject* utf = NULL; if (PyUnicode_Check(sval)) { utf = PyUnicode_AsEncodedString(sval, "utf-8", "ignore"); sval = utf; } - res = PyString_AsStringAndSize(sval, &buf, &len); + res = PyBytes_AsStringAndSize(sval, &buf, &len); if (res == -1) { res = 0; goto out; } - if (!bser_append(bser, &bser_string_hdr, sizeof(bser_string_hdr))) { + if (!bser_append(bser, &bser_bytestring_hdr, sizeof(bser_bytestring_hdr))) { res = 0; goto out; } @@ -341,8 +397,7 @@ return res; } -static int bser_recursive(bser_t *bser, PyObject *val) -{ +static int bser_recursive(bser_t* bser, PyObject* val) { if (PyBool_Check(val)) { if (val == Py_True) { return bser_append(bser, &bser_true, sizeof(bser_true)); @@ -354,19 +409,21 @@ return bser_append(bser, &bser_null, sizeof(bser_null)); } +// Python 3 has one integer type. +#if PY_MAJOR_VERSION < 3 if (PyInt_Check(val)) { return bser_long(bser, PyInt_AS_LONG(val)); } +#endif // PY_MAJOR_VERSION < 3 if (PyLong_Check(val)) { return bser_long(bser, PyLong_AsLongLong(val)); } - if (PyString_Check(val) || PyUnicode_Check(val)) { - return bser_string(bser, val); + if (PyBytes_Check(val) || PyUnicode_Check(val)) { + return bser_bytestring(bser, val); } - if (PyFloat_Check(val)) { double dval = PyFloat_AS_DOUBLE(val); char sz = BSER_REAL; @@ -390,7 +447,7 @@ } for (i = 0; i < len; i++) { - PyObject *ele = PyList_GET_ITEM(val, i); + PyObject* ele = PyList_GET_ITEM(val, i); if (!bser_recursive(bser, ele)) { return 0; @@ -412,7 +469,7 @@ } for (i = 0; i < len; i++) { - PyObject *ele = PyTuple_GET_ITEM(val, i); + PyObject* ele = PyTuple_GET_ITEM(val, i); if (!bser_recursive(bser, ele)) { return 0; @@ -436,7 +493,7 @@ } while (PyDict_Next(val, &pos, &key, &ele)) { - if (!bser_string(bser, key)) { + if (!bser_bytestring(bser, key)) { return 0; } if (!bser_recursive(bser, ele)) { @@ -451,17 +508,25 @@ return 0; } -static PyObject *bser_dumps(PyObject *self, PyObject *args) -{ +static PyObject* bser_dumps(PyObject* self, PyObject* args, PyObject* kw) { PyObject *val = NULL, *res; bser_t bser; - uint32_t len; + uint32_t len, bser_version = 1, bser_capabilities = 0; - if (!PyArg_ParseTuple(args, "O", &val)) { + static char* kw_list[] = {"val", "version", "capabilities", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + args, + kw, + "O|ii:dumps", + kw_list, + &val, + &bser_version, + &bser_capabilities)) { return NULL; } - if (!bser_init(&bser)) { + if (!bser_init(&bser, bser_version, bser_capabilities)) { return PyErr_NoMemory(); } @@ -475,19 +540,25 @@ } // Now fill in the overall length - len = bser.wpos - (sizeof(EMPTY_HEADER) - 1); - memcpy(bser.buf + 3, &len, sizeof(len)); + if (bser_version == 1) { + len = bser.wpos - (sizeof(EMPTY_HEADER) - 1); + memcpy(bser.buf + 3, &len, sizeof(len)); + } else { + len = bser.wpos - (sizeof(EMPTY_HEADER_V2) - 1); + // The BSER capabilities block comes before the PDU length + memcpy(bser.buf + 2, &bser_capabilities, sizeof(bser_capabilities)); + memcpy(bser.buf + 7, &len, sizeof(len)); + } - res = PyString_FromStringAndSize(bser.buf, bser.wpos); + res = PyBytes_FromStringAndSize(bser.buf, bser.wpos); bser_dtor(&bser); return res; } -int bunser_int(const char **ptr, const char *end, int64_t *val) -{ +int bunser_int(const char** ptr, const char* end, int64_t* val) { int needed; - const char *buf = *ptr; + const char* buf = *ptr; int8_t i8; int16_t i16; int32_t i32; @@ -507,8 +578,8 @@ needed = 9; break; default: - PyErr_Format(PyExc_ValueError, - "invalid bser int encoding 0x%02x", buf[0]); + PyErr_Format( + PyExc_ValueError, "invalid bser int encoding 0x%02x", buf[0]); return 0; } if (end - buf < needed) { @@ -538,10 +609,12 @@ } } -static int bunser_string(const char **ptr, const char *end, - const char **start, int64_t *len) -{ - const char *buf = *ptr; +static int bunser_bytestring( + const char** ptr, + const char* end, + const char** start, + int64_t* len) { + const char* buf = *ptr; // skip string marker buf++; @@ -559,11 +632,12 @@ return 1; } -static PyObject *bunser_array(const char **ptr, const char *end, int mutable) -{ - const char *buf = *ptr; +static PyObject* +bunser_array(const char** ptr, const char* end, const unser_ctx_t* ctx) { + const char* buf = *ptr; int64_t nitems, i; - PyObject *res; + int mutable = ctx->mutable; + PyObject* res; // skip array header buf++; @@ -584,7 +658,7 @@ } for (i = 0; i < nitems; i++) { - PyObject *ele = bser_loads_recursive(ptr, end, mutable); + PyObject* ele = bser_loads_recursive(ptr, end, ctx); if (!ele) { Py_DECREF(res); @@ -602,13 +676,13 @@ return res; } -static PyObject *bunser_object(const char **ptr, const char *end, - int mutable) -{ - const char *buf = *ptr; +static PyObject* +bunser_object(const char** ptr, const char* end, const unser_ctx_t* ctx) { + const char* buf = *ptr; int64_t nitems, i; - PyObject *res; - bserObject *obj; + int mutable = ctx->mutable; + PyObject* res; + bserObject* obj; // skip array header buf++; @@ -627,12 +701,12 @@ } for (i = 0; i < nitems; i++) { - const char *keystr; + const char* keystr; int64_t keylen; - PyObject *key; - PyObject *ele; + PyObject* key; + PyObject* ele; - if (!bunser_string(ptr, end, &keystr, &keylen)) { + if (!bunser_bytestring(ptr, end, &keystr, &keylen)) { Py_DECREF(res); return NULL; } @@ -643,13 +717,24 @@ return NULL; } - key = PyString_FromStringAndSize(keystr, (Py_ssize_t)keylen); + if (mutable) { + // This will interpret the key as UTF-8. + key = PyUnicode_FromStringAndSize(keystr, (Py_ssize_t)keylen); + } else { + // For immutable objects we'll manage key lookups, so we can avoid going + // through the Unicode APIs. This avoids a potentially expensive and + // definitely unnecessary conversion to UTF-16 and back for Python 2. + // TODO: On Python 3 the Unicode APIs are smarter: we might be able to use + // Unicode keys there without an appreciable performance loss. + key = PyBytes_FromStringAndSize(keystr, (Py_ssize_t)keylen); + } + if (!key) { Py_DECREF(res); return NULL; } - ele = bser_loads_recursive(ptr, end, mutable); + ele = bser_loads_recursive(ptr, end, ctx); if (!ele) { Py_DECREF(key); @@ -671,14 +756,24 @@ return res; } -static PyObject *bunser_template(const char **ptr, const char *end, - int mutable) -{ - const char *buf = *ptr; +static PyObject* +bunser_template(const char** ptr, const char* end, const unser_ctx_t* ctx) { + const char* buf = *ptr; int64_t nitems, i; - PyObject *arrval; - PyObject *keys; + int mutable = ctx->mutable; + PyObject* arrval; + PyObject* keys; Py_ssize_t numkeys, keyidx; + unser_ctx_t keys_ctx = {0}; + if (mutable) { + keys_ctx.mutable = 1; + // Decode keys as UTF-8 in this case. + keys_ctx.value_encoding = "utf-8"; + keys_ctx.value_errors = "strict"; + } else { + // Treat keys as bytestrings in this case -- we'll do Unicode conversions at + // lookup time. + } if (buf[1] != BSER_ARRAY) { PyErr_Format(PyExc_ValueError, "Expect ARRAY to follow TEMPLATE"); @@ -689,8 +784,9 @@ buf++; *ptr = buf; - // Load template keys - keys = bunser_array(ptr, end, mutable); + // Load template keys. + // For keys we don't want to do any decoding right now. + keys = bunser_array(ptr, end, &keys_ctx); if (!keys) { return NULL; } @@ -716,8 +812,8 @@ } for (i = 0; i < nitems; i++) { - PyObject *dict = NULL; - bserObject *obj = NULL; + PyObject* dict = NULL; + bserObject* obj = NULL; if (mutable) { dict = PyDict_New(); @@ -731,22 +827,22 @@ dict = (PyObject*)obj; } if (!dict) { -fail: + fail: Py_DECREF(keys); Py_DECREF(arrval); return NULL; } for (keyidx = 0; keyidx < numkeys; keyidx++) { - PyObject *key; - PyObject *ele; + PyObject* key; + PyObject* ele; if (**ptr == BSER_SKIP) { *ptr = *ptr + 1; ele = Py_None; Py_INCREF(ele); } else { - ele = bser_loads_recursive(ptr, end, mutable); + ele = bser_loads_recursive(ptr, end, ctx); } if (!ele) { @@ -772,34 +868,38 @@ return arrval; } -static PyObject *bser_loads_recursive(const char **ptr, const char *end, - int mutable) -{ - const char *buf = *ptr; +static PyObject* bser_loads_recursive( + const char** ptr, + const char* end, + const unser_ctx_t* ctx) { + const char* buf = *ptr; switch (buf[0]) { case BSER_INT8: case BSER_INT16: case BSER_INT32: - case BSER_INT64: - { - int64_t ival; - if (!bunser_int(ptr, end, &ival)) { - return NULL; - } - if (ival < LONG_MIN || ival > LONG_MAX) { - return PyLong_FromLongLong(ival); - } - return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t)); - } - - case BSER_REAL: - { - double dval; - memcpy(&dval, buf + 1, sizeof(dval)); - *ptr = buf + 1 + sizeof(double); - return PyFloat_FromDouble(dval); + case BSER_INT64: { + int64_t ival; + if (!bunser_int(ptr, end, &ival)) { + return NULL; + } +// Python 3 has one integer type. +#if PY_MAJOR_VERSION >= 3 + return PyLong_FromLongLong(ival); +#else + if (ival < LONG_MIN || ival > LONG_MAX) { + return PyLong_FromLongLong(ival); } + return PyInt_FromSsize_t(Py_SAFE_DOWNCAST(ival, int64_t, Py_ssize_t)); +#endif // PY_MAJOR_VERSION >= 3 + } + + case BSER_REAL: { + double dval; + memcpy(&dval, buf + 1, sizeof(dval)); + *ptr = buf + 1 + sizeof(double); + return PyFloat_FromDouble(dval); + } case BSER_TRUE: *ptr = buf + 1; @@ -816,31 +916,51 @@ Py_INCREF(Py_None); return Py_None; - case BSER_STRING: - { - const char *start; - int64_t len; - - if (!bunser_string(ptr, end, &start, &len)) { - return NULL; - } - - if (len > LONG_MAX) { - PyErr_Format(PyExc_ValueError, "string too long for python"); - return NULL; - } + case BSER_BYTESTRING: { + const char* start; + int64_t len; + + if (!bunser_bytestring(ptr, end, &start, &len)) { + return NULL; + } + + if (len > LONG_MAX) { + PyErr_Format(PyExc_ValueError, "string too long for python"); + return NULL; + } + + if (ctx->value_encoding != NULL) { + return PyUnicode_Decode( + start, (long)len, ctx->value_encoding, ctx->value_errors); + } else { + return PyBytes_FromStringAndSize(start, (long)len); + } + } + + case BSER_UTF8STRING: { + const char* start; + int64_t len; - return PyString_FromStringAndSize(start, (long)len); + if (!bunser_bytestring(ptr, end, &start, &len)) { + return NULL; } + if (len > LONG_MAX) { + PyErr_Format(PyExc_ValueError, "string too long for python"); + return NULL; + } + + return PyUnicode_Decode(start, (long)len, "utf-8", "strict"); + } + case BSER_ARRAY: - return bunser_array(ptr, end, mutable); + return bunser_array(ptr, end, ctx); case BSER_OBJECT: - return bunser_object(ptr, end, mutable); + return bunser_object(ptr, end, ctx); case BSER_TEMPLATE: - return bunser_template(ptr, end, mutable); + return bunser_template(ptr, end, ctx); default: PyErr_Format(PyExc_ValueError, "unhandled bser opcode 0x%02x", buf[0]); @@ -849,102 +969,244 @@ return NULL; } -// Expected use case is to read a packet from the socket and -// then call bser.pdu_len on the packet. It returns the total -// length of the entire response that the peer is sending, -// including the bytes already received. This allows the client -// to compute the data size it needs to read before it can -// decode the data -static PyObject *bser_pdu_len(PyObject *self, PyObject *args) -{ - const char *start = NULL; - const char *data = NULL; - int datalen = 0; - const char *end; - int64_t expected_len, total_len; - - if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) { - return NULL; - } - data = start; - end = data + datalen; +static int _pdu_info_helper( + const char* data, + const char* end, + uint32_t* bser_version_out, + uint32_t* bser_capabilities_out, + int64_t* expected_len_out, + off_t* position_out) { + uint32_t bser_version; + uint32_t bser_capabilities = 0; + int64_t expected_len; + const char* start; + start = data; // Validate the header and length - if (memcmp(data, EMPTY_HEADER, 2) != 0) { + if (memcmp(data, EMPTY_HEADER, 2) == 0) { + bser_version = 1; + } else if (memcmp(data, EMPTY_HEADER_V2, 2) == 0) { + bser_version = 2; + } else { PyErr_SetString(PyExc_ValueError, "invalid bser header"); - return NULL; + return 0; } data += 2; + if (bser_version == 2) { + // Expect an integer telling us what capabilities are supported by the + // remote server (currently unused). + if (!memcpy(&bser_capabilities, &data, sizeof(bser_capabilities))) { + return 0; + } + data += sizeof(bser_capabilities); + } + // Expect an integer telling us how big the rest of the data // should be if (!bunser_int(&data, end, &expected_len)) { - return NULL; + return 0; } - total_len = expected_len + (data - start); - if (total_len > LONG_MAX) { - return PyLong_FromLongLong(total_len); - } - return PyInt_FromLong((long)total_len); + *bser_version_out = bser_version; + *bser_capabilities_out = (uint32_t)bser_capabilities; + *expected_len_out = expected_len; + *position_out = (off_t)(data - start); + return 1; } -static PyObject *bser_loads(PyObject *self, PyObject *args) -{ - const char *data = NULL; +// This function parses the PDU header and provides info about the packet +// Returns false if unsuccessful +static int pdu_info_helper( + PyObject* self, + PyObject* args, + uint32_t* bser_version_out, + uint32_t* bser_capabilities_out, + int64_t* total_len_out) { + const char* start = NULL; + const char* data = NULL; int datalen = 0; - const char *end; + const char* end; int64_t expected_len; - int mutable = 1; - PyObject *mutable_obj = NULL; + off_t position; - if (!PyArg_ParseTuple(args, "s#|O:loads", &data, &datalen, &mutable_obj)) { - return NULL; + if (!PyArg_ParseTuple(args, "s#", &start, &datalen)) { + return 0; } - if (mutable_obj) { - mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0; + data = start; + end = data + datalen; + + if (!_pdu_info_helper( + data, + end, + bser_version_out, + bser_capabilities_out, + &expected_len, + &position)) { + return 0; } + *total_len_out = (int64_t)(expected_len + position); + return 1; +} - end = data + datalen; +// Expected use case is to read a packet from the socket and then call +// bser.pdu_info on the packet. It returns the BSER version, BSER capabilities, +// and the total length of the entire response that the peer is sending, +// including the bytes already received. This allows the client to compute the +// data size it needs to read before it can decode the data. +static PyObject* bser_pdu_info(PyObject* self, PyObject* args) { + uint32_t version, capabilities; + int64_t total_len; + if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) { + return NULL; + } + return Py_BuildValue("kkL", version, capabilities, total_len); +} - // Validate the header and length - if (memcmp(data, EMPTY_HEADER, 2) != 0) { - PyErr_SetString(PyExc_ValueError, "invalid bser header"); +static PyObject* bser_pdu_len(PyObject* self, PyObject* args) { + uint32_t version, capabilities; + int64_t total_len; + if (!pdu_info_helper(self, args, &version, &capabilities, &total_len)) { return NULL; } + return Py_BuildValue("L", total_len); +} - data += 2; +static PyObject* bser_loads(PyObject* self, PyObject* args, PyObject* kw) { + const char* data = NULL; + int datalen = 0; + const char* start; + const char* end; + int64_t expected_len; + off_t position; + PyObject* mutable_obj = NULL; + const char* value_encoding = NULL; + const char* value_errors = NULL; + unser_ctx_t ctx = {1, 0}; + + static char* kw_list[] = { + "buf", "mutable", "value_encoding", "value_errors", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + args, + kw, + "s#|Ozz:loads", + kw_list, + &start, + &datalen, + &mutable_obj, + &value_encoding, + &value_errors)) { + return NULL; + } - // Expect an integer telling us how big the rest of the data - // should be - if (!bunser_int(&data, end, &expected_len)) { + if (mutable_obj) { + ctx.mutable = PyObject_IsTrue(mutable_obj) > 0 ? 1 : 0; + } + ctx.value_encoding = value_encoding; + if (value_encoding == NULL) { + ctx.value_errors = NULL; + } else if (value_errors == NULL) { + ctx.value_errors = "strict"; + } else { + ctx.value_errors = value_errors; + } + data = start; + end = data + datalen; + + if (!_pdu_info_helper( + data, + end, + &ctx.bser_version, + &ctx.bser_capabilities, + &expected_len, + &position)) { return NULL; } + data = start + position; // Verify if (expected_len + data != end) { PyErr_SetString(PyExc_ValueError, "bser data len != header len"); return NULL; } - return bser_loads_recursive(&data, end, mutable); + return bser_loads_recursive(&data, end, &ctx); } +static PyObject* bser_load(PyObject* self, PyObject* args, PyObject* kw) { + PyObject *load, *string; + PyObject* fp = NULL; + PyObject* mutable_obj = NULL; + const char* value_encoding = NULL; + const char* value_errors = NULL; + + static char* kw_list[] = { + "fp", "mutable", "value_encoding", "value_errors", NULL}; + + if (!PyArg_ParseTupleAndKeywords( + args, + kw, + "OOzz:load", + kw_list, + &fp, + &mutable_obj, + &value_encoding, + &value_errors)) { + return NULL; + } + + load = PyImport_ImportModule("pywatchman.load"); + if (load == NULL) { + return NULL; + } + string = PyObject_CallMethod( + load, "load", "OOzz", fp, mutable_obj, value_encoding, value_errors); + Py_DECREF(load); + return string; +} + +// clang-format off static PyMethodDef bser_methods[] = { - {"loads", bser_loads, METH_VARARGS, "Deserialize string."}, - {"pdu_len", bser_pdu_len, METH_VARARGS, "Extract PDU length."}, - {"dumps", bser_dumps, METH_VARARGS, "Serialize string."}, + {"loads", (PyCFunction)bser_loads, METH_VARARGS | METH_KEYWORDS, + "Deserialize string."}, + {"load", (PyCFunction)bser_load, METH_VARARGS | METH_KEYWORDS, + "Deserialize a file object"}, + {"pdu_info", (PyCFunction)bser_pdu_info, METH_VARARGS, + "Extract PDU information."}, + {"pdu_len", (PyCFunction)bser_pdu_len, METH_VARARGS, + "Extract total PDU length."}, + {"dumps", (PyCFunction)bser_dumps, METH_VARARGS | METH_KEYWORDS, + "Serialize string."}, {NULL, NULL, 0, NULL} }; -PyMODINIT_FUNC initbser(void) -{ +#if PY_MAJOR_VERSION >= 3 +static struct PyModuleDef bser_module = { + PyModuleDef_HEAD_INIT, + "bser", + "Efficient encoding and decoding of BSER.", + -1, + bser_methods +}; +// clang-format on + +PyMODINIT_FUNC PyInit_bser(void) { + PyObject* mod; + + mod = PyModule_Create(&bser_module); + PyType_Ready(&bserObjectType); + + return mod; +} +#else + +PyMODINIT_FUNC initbser(void) { (void)Py_InitModule("bser", bser_methods); PyType_Ready(&bserObjectType); } +#endif // PY_MAJOR_VERSION >= 3 /* vim:ts=2:sw=2:et: */ - -// no-check-code -- this is a 3rd party library diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/capabilities.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/capabilities.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/capabilities.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/capabilities.py 2017-01-18 16:54:51.000000000 +0000 @@ -26,6 +26,11 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# no unicode literals + import re def parse_version(vstr): @@ -65,5 +70,3 @@ vers['error'] = 'client required capability `' + name + \ '` is not supported by this server' return vers - -# no-check-code -- this is a 3rd party library diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/compat.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/compat.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/compat.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/compat.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,65 @@ +# Copyright 2016-present Facebook, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name Facebook nor the names of its contributors may be used to +# endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# no unicode literals + +'''Compatibility module across Python 2 and 3.''' + +import sys + +PYTHON3 = sys.version_info >= (3, 0) + +# This is adapted from https://bitbucket.org/gutworth/six, and used under the +# MIT license. See LICENSE for a full copyright notice. +if PYTHON3: + def reraise(tp, value, tb=None): + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None +else: + exec(''' +def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +'''.strip()) + +if PYTHON3: + UNICODE = str +else: + UNICODE = unicode diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/encoding.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/encoding.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/encoding.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/encoding.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,73 @@ +# Copyright 2016-present Facebook, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name Facebook nor the names of its contributors may be used to +# endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# no unicode literals + +'''Module to deal with filename encoding on the local system, as returned by +Watchman.''' + +import sys + +from . import ( + compat, +) + +if compat.PYTHON3: + default_local_errors = 'surrogateescape' + + def get_local_encoding(): + if sys.platform == 'win32': + # Watchman always returns UTF-8 encoded strings on Windows. + return 'utf-8' + # On the Python 3 versions we support, sys.getfilesystemencoding never + # returns None. + return sys.getfilesystemencoding() +else: + # Python 2 doesn't support surrogateescape, so use 'strict' by + # default. Users can register a custom surrogateescape error handler and use + # that if they so desire. + default_local_errors = 'strict' + + def get_local_encoding(): + if sys.platform == 'win32': + # Watchman always returns UTF-8 encoded strings on Windows. + return 'utf-8' + fsencoding = sys.getfilesystemencoding() + if fsencoding is None: + # This is very unlikely to happen, but if it does, just use UTF-8 + fsencoding = 'utf-8' + return fsencoding + +def encode_local(s): + return s.encode(get_local_encoding(), default_local_errors) + +def decode_local(bs): + return bs.decode(get_local_encoding(), default_local_errors) diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/__init__.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/__init__.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/__init__.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/__init__.py 2017-01-18 16:54:51.000000000 +0000 @@ -26,9 +26,14 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import os -import errno +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# no unicode literals + +import inspect import math +import os import socket import subprocess import time @@ -36,11 +41,20 @@ # Sometimes it's really hard to get Python extensions to compile, # so fall back to a pure Python implementation. try: - import bser + from . import bser + # Demandimport causes modules to be loaded lazily. Force the load now + # so that we can fall back on pybser if bser doesn't exist + bser.pdu_info except ImportError: - import pybser as bser + from . import pybser as bser + +from . import ( + capabilities, + compat, + encoding, + load, +) -import capabilities if os.name == 'nt': import ctypes @@ -55,18 +69,29 @@ FORMAT_MESSAGE_FROM_SYSTEM = 0x00001000 FORMAT_MESSAGE_ALLOCATE_BUFFER = 0x00000100 FORMAT_MESSAGE_IGNORE_INSERTS = 0x00000200 + WAIT_FAILED = 0xFFFFFFFF WAIT_TIMEOUT = 0x00000102 WAIT_OBJECT_0 = 0x00000000 - ERROR_IO_PENDING = 997 + WAIT_IO_COMPLETION = 0x000000C0 + INFINITE = 0xFFFFFFFF + + # Overlapped I/O operation is in progress. (997) + ERROR_IO_PENDING = 0x000003E5 + + # The pointer size follows the architecture + # We use WPARAM since this type is already conditionally defined + ULONG_PTR = ctypes.wintypes.WPARAM class OVERLAPPED(ctypes.Structure): _fields_ = [ - ("Internal", wintypes.ULONG), ("InternalHigh", wintypes.ULONG), + ("Internal", ULONG_PTR), ("InternalHigh", ULONG_PTR), ("Offset", wintypes.DWORD), ("OffsetHigh", wintypes.DWORD), ("hEvent", wintypes.HANDLE) ] def __init__(self): + self.Internal = 0 + self.InternalHigh = 0 self.Offset = 0 self.OffsetHigh = 0 self.hEvent = 0 @@ -97,6 +122,10 @@ GetLastError.argtypes = [] GetLastError.restype = wintypes.DWORD + SetLastError = ctypes.windll.kernel32.SetLastError + SetLastError.argtypes = [wintypes.DWORD] + SetLastError.restype = None + FormatMessage = ctypes.windll.kernel32.FormatMessageA FormatMessage.argtypes = [wintypes.DWORD, wintypes.LPVOID, wintypes.DWORD, wintypes.DWORD, ctypes.POINTER(wintypes.LPSTR), @@ -105,12 +134,30 @@ LocalFree = ctypes.windll.kernel32.LocalFree - GetOverlappedResultEx = ctypes.windll.kernel32.GetOverlappedResultEx - GetOverlappedResultEx.argtypes = [wintypes.HANDLE, - ctypes.POINTER(OVERLAPPED), LPDWORD, - wintypes.DWORD, wintypes.BOOL] - GetOverlappedResultEx.restype = wintypes.BOOL + GetOverlappedResult = ctypes.windll.kernel32.GetOverlappedResult + GetOverlappedResult.argtypes = [wintypes.HANDLE, + ctypes.POINTER(OVERLAPPED), LPDWORD, + wintypes.BOOL] + GetOverlappedResult.restype = wintypes.BOOL + + GetOverlappedResultEx = getattr(ctypes.windll.kernel32, + 'GetOverlappedResultEx', None) + if GetOverlappedResultEx is not None: + GetOverlappedResultEx.argtypes = [wintypes.HANDLE, + ctypes.POINTER(OVERLAPPED), LPDWORD, + wintypes.DWORD, wintypes.BOOL] + GetOverlappedResultEx.restype = wintypes.BOOL + + WaitForSingleObjectEx = ctypes.windll.kernel32.WaitForSingleObjectEx + WaitForSingleObjectEx.argtypes = [wintypes.HANDLE, wintypes.DWORD, wintypes.BOOL] + WaitForSingleObjectEx.restype = wintypes.DWORD + + CreateEvent = ctypes.windll.kernel32.CreateEventA + CreateEvent.argtypes = [LPDWORD, wintypes.BOOL, wintypes.BOOL, + wintypes.LPSTR] + CreateEvent.restype = wintypes.HANDLE + # Windows Vista is the minimum supported client for CancelIoEx. CancelIoEx = ctypes.windll.kernel32.CancelIoEx CancelIoEx.argtypes = [wintypes.HANDLE, ctypes.POINTER(OVERLAPPED)] CancelIoEx.restype = wintypes.BOOL @@ -132,8 +179,47 @@ pass +def _win32_strerror(err): + """ expand a win32 error code into a human readable message """ + + # FormatMessage will allocate memory and assign it here + buf = ctypes.c_char_p() + FormatMessage( + FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None) + try: + return buf.value + finally: + LocalFree(buf) + + class WatchmanError(Exception): - pass + def __init__(self, msg=None, cmd=None): + self.msg = msg + self.cmd = cmd + + def setCommand(self, cmd): + self.cmd = cmd + + def __str__(self): + if self.cmd: + return '%s, while executing %s' % (self.msg, self.cmd) + return self.msg + + +class WatchmanEnvironmentError(WatchmanError): + def __init__(self, msg, errno, errmsg, cmd=None): + super(WatchmanEnvironmentError, self).__init__( + '{0}: errno={1} errmsg={2}'.format(msg, errno, errmsg), + cmd) + + +class SocketConnectError(WatchmanError): + def __init__(self, sockpath, exc): + super(SocketConnectError, self).__init__( + 'unable to connect to %s: %s' % (sockpath, exc)) + self.sockpath = sockpath + self.exc = exc class SocketTimeout(WatchmanError): @@ -151,19 +237,11 @@ self.msg is the message returned by watchman. """ - def __init__(self, msg, cmd=None): - self.msg = msg - self.cmd = cmd - super(CommandError, self).__init__('watchman command error: %s' % msg) - - def setCommand(self, cmd): - self.cmd = cmd - - def __str__(self): - if self.cmd: - return '%s, while executing %s' % (self.msg, self.cmd) - return self.msg + super(CommandError, self).__init__( + 'watchman command error: %s' % (msg, ), + cmd, + ) class Transport(object): @@ -195,16 +273,16 @@ # Buffer may already have a line if we've received unilateral # response(s) from the server - if len(self.buf) == 1 and "\n" in self.buf[0]: - (line, b) = self.buf[0].split("\n", 1) + if len(self.buf) == 1 and b"\n" in self.buf[0]: + (line, b) = self.buf[0].split(b"\n", 1) self.buf = [b] return line while True: b = self.readBytes(4096) - if "\n" in b: - result = ''.join(self.buf) - (line, b) = b.split("\n", 1) + if b"\n" in b: + result = b''.join(self.buf) + (line, b) = b.split(b"\n", 1) self.buf = [b] return result + line self.buf.append(b) @@ -241,8 +319,8 @@ sock.connect(self.sockpath) self.sock = sock except socket.error as e: - raise WatchmanError('unable to connect to %s: %s' % - (self.sockpath, e)) + sock.close() + raise SocketConnectError(self.sockpath, e) def close(self): self.sock.close() @@ -268,6 +346,46 @@ raise SocketTimeout('timed out sending query command') +def _get_overlapped_result_ex_impl(pipe, olap, nbytes, millis, alertable): + """ Windows 7 and earlier does not support GetOverlappedResultEx. The + alternative is to use GetOverlappedResult and wait for read or write + operation to complete. This is done be using CreateEvent and + WaitForSingleObjectEx. CreateEvent, WaitForSingleObjectEx + and GetOverlappedResult are all part of Windows API since WindowsXP. + This is the exact same implementation that can be found in the watchman + source code (see get_overlapped_result_ex_impl in stream_win.c). This + way, maintenance should be simplified. + """ + log('Preparing to wait for maximum %dms', millis ) + if millis != 0: + waitReturnCode = WaitForSingleObjectEx(olap.hEvent, millis, alertable) + if waitReturnCode == WAIT_OBJECT_0: + # Event is signaled, overlapped IO operation result should be available. + pass + elif waitReturnCode == WAIT_IO_COMPLETION: + # WaitForSingleObjectEx returnes because the system added an I/O completion + # routine or an asynchronous procedure call (APC) to the thread queue. + SetLastError(WAIT_IO_COMPLETION) + pass + elif waitReturnCode == WAIT_TIMEOUT: + # We reached the maximum allowed wait time, the IO operation failed + # to complete in timely fashion. + SetLastError(WAIT_TIMEOUT) + return False + elif waitReturnCode == WAIT_FAILED: + # something went wrong calling WaitForSingleObjectEx + err = GetLastError() + log('WaitForSingleObjectEx failed: %s', _win32_strerror(err)) + return False + else: + # unexpected situation deserving investigation. + err = GetLastError() + log('Unexpected error: %s', _win32_strerror(err)) + return False + + return GetOverlappedResult(pipe, olap, nbytes, False) + + class WindowsNamedPipeTransport(Transport): """ connect to a named pipe """ @@ -284,28 +402,35 @@ self._raise_win_err('failed to open pipe %s' % sockpath, GetLastError()) - def _win32_strerror(self, err): - """ expand a win32 error code into a human readable message """ - - # FormatMessage will allocate memory and assign it here - buf = ctypes.c_char_p() - FormatMessage( - FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER - | FORMAT_MESSAGE_IGNORE_INSERTS, None, err, 0, buf, 0, None) - try: - return buf.value - finally: - LocalFree(buf) + # event for the overlapped I/O operations + self._waitable = CreateEvent(None, True, False, None) + if self._waitable is None: + self._raise_win_err('CreateEvent failed', GetLastError()) + + self._get_overlapped_result_ex = GetOverlappedResultEx + if (os.getenv('WATCHMAN_WIN7_COMPAT') == '1' or + self._get_overlapped_result_ex is None): + self._get_overlapped_result_ex = _get_overlapped_result_ex_impl def _raise_win_err(self, msg, err): raise IOError('%s win32 error code: %d %s' % - (msg, err, self._win32_strerror(err))) + (msg, err, _win32_strerror(err))) def close(self): if self.pipe: + log('Closing pipe') CloseHandle(self.pipe) self.pipe = None + if self._waitable is not None: + # We release the handle for the event + CloseHandle(self._waitable) + self._waitable = None + + def setTimeout(self, value): + # convert to milliseconds + self.timeout = int(value * 1000) + def readBytes(self, size): """ A read can block for an unbounded amount of time, even if the kernel reports that the pipe handle is signalled, so we need to @@ -325,6 +450,7 @@ # We need to initiate a read buf = ctypes.create_string_buffer(size) olap = OVERLAPPED() + olap.hEvent = self._waitable log('made read buff of size %d', size) @@ -339,8 +465,9 @@ GetLastError()) nread = wintypes.DWORD() - if not GetOverlappedResultEx(self.pipe, olap, nread, - 0 if immediate else self.timeout, True): + if not self._get_overlapped_result_ex(self.pipe, olap, nread, + 0 if immediate else self.timeout, + True): err = GetLastError() CancelIoEx(self.pipe, olap) @@ -374,6 +501,8 @@ def write(self, data): olap = OVERLAPPED() + olap.hEvent = self._waitable + immediate = WriteFile(self.pipe, ctypes.c_char_p(data), len(data), None, olap) @@ -385,8 +514,10 @@ # Obtain results, waiting if needed nwrote = wintypes.DWORD() - if GetOverlappedResultEx(self.pipe, olap, nwrote, 0 if immediate else - self.timeout, True): + if self._get_overlapped_result_ex(self.pipe, olap, nwrote, + 0 if immediate else self.timeout, + True): + log('made write of %d bytes', nwrote.value) return nwrote.value err = GetLastError() @@ -430,7 +561,10 @@ def close(self): if self.proc: - self.proc.kill() + if self.proc.pid is not None: + self.proc.kill() + self.proc.stdin.close() + self.proc.stdout.close() self.proc = None def _connect(self): @@ -438,7 +572,7 @@ return self.proc args = [ 'watchman', - '--sockname={}'.format(self.sockpath), + '--sockname={0}'.format(self.sockpath), '--logfile=/BOGUS', '--statefile=/BOGUS', '--no-spawn', @@ -460,8 +594,8 @@ def write(self, data): if self.closed: + self.close() self.closed = False - self.proc = None self._connect() res = self.proc.stdin.write(data) self.proc.stdin.close() @@ -473,21 +607,21 @@ """ use the BSER encoding. This is the default, preferred codec """ def _loads(self, response): - return bser.loads(response) + return bser.loads(response) # Defaults to BSER v1 def receive(self): buf = [self.transport.readBytes(sniff_len)] if not buf[0]: raise WatchmanError('empty watchman response') - elen = bser.pdu_len(buf[0]) + _1, _2, elen = bser.pdu_info(buf[0]) rlen = len(buf[0]) while elen > rlen: buf.append(self.transport.readBytes(elen - rlen)) rlen += len(buf[-1]) - response = ''.join(buf) + response = b''.join(buf) try: res = self._loads(response) return res @@ -495,7 +629,7 @@ raise WatchmanError('watchman response decode error: %s' % e) def send(self, *args): - cmd = bser.dumps(*args) + cmd = bser.dumps(*args) # Defaults to BSER v1 self.transport.write(cmd) @@ -504,7 +638,64 @@ immutable object support """ def _loads(self, response): - return bser.loads(response, False) + return bser.loads(response, False) # Defaults to BSER v1 + + +class Bser2WithFallbackCodec(BserCodec): + """ use BSER v2 encoding """ + + def __init__(self, transport): + super(Bser2WithFallbackCodec, self).__init__(transport) + # Once the server advertises support for bser-v2 we should switch this + # to 'required' on Python 3. + self.send(["version", {"optional": ["bser-v2"]}]) + + capabilities = self.receive() + + if 'error' in capabilities: + raise Exception('Unsupported BSER version') + + if capabilities['capabilities']['bser-v2']: + self.bser_version = 2 + self.bser_capabilities = 0 + else: + self.bser_version = 1 + self.bser_capabilities = 0 + + def _loads(self, response): + return bser.loads(response) + + def receive(self): + buf = [self.transport.readBytes(sniff_len)] + if not buf[0]: + raise WatchmanError('empty watchman response') + + recv_bser_version, recv_bser_capabilities, elen = bser.pdu_info(buf[0]) + + if hasattr(self, 'bser_version'): + # Readjust BSER version and capabilities if necessary + self.bser_version = max(self.bser_version, recv_bser_version) + self.capabilities = self.bser_capabilities & recv_bser_capabilities + + rlen = len(buf[0]) + while elen > rlen: + buf.append(self.transport.readBytes(elen - rlen)) + rlen += len(buf[-1]) + + response = b''.join(buf) + try: + res = self._loads(response) + return res + except ValueError as e: + raise WatchmanError('watchman response decode error: %s' % e) + + def send(self, *args): + if hasattr(self, 'bser_version'): + cmd = bser.dumps(*args, version=self.bser_version, + capabilities=self.bser_capabilities) + else: + cmd = bser.dumps(*args) + self.transport.write(cmd) class JsonCodec(Codec): @@ -520,6 +711,13 @@ def receive(self): line = self.transport.readLine() try: + # In Python 3, json.loads is a transformation from Unicode string to + # objects possibly containing Unicode strings. We typically expect + # the JSON blob to be ASCII-only with non-ASCII characters escaped, + # but it's possible we might get non-ASCII bytes that are valid + # UTF-8. + if compat.PYTHON3: + line = line.decode('utf-8') return self.json.loads(line) except Exception as e: print(e, line) @@ -527,7 +725,12 @@ def send(self, *args): cmd = self.json.dumps(*args) - self.transport.write(cmd + "\n") + # In Python 3, json.dumps is a transformation from objects possibly + # containing Unicode strings to Unicode string. Even with (the default) + # ensure_ascii=True, dumps returns a Unicode string. + if compat.PYTHON3: + cmd = cmd.encode('ascii') + self.transport.write(cmd + b"\n") class client(object): @@ -556,22 +759,27 @@ self.timeout = timeout self.useImmutableBser = useImmutableBser - transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local' - if transport == 'local' and os.name == 'nt': - self.transport = WindowsNamedPipeTransport - elif transport == 'local': - self.transport = UnixSocketTransport - elif transport == 'cli': - self.transport = CLIProcessTransport - if sendEncoding is None: - sendEncoding = 'json' - if recvEncoding is None: - recvEncoding = sendEncoding + if inspect.isclass(transport) and issubclass(transport, Transport): + self.transport = transport else: - raise WatchmanError('invalid transport %s' % transport) - - sendEncoding = sendEncoding or os.getenv('WATCHMAN_ENCODING') or 'bser' - recvEncoding = recvEncoding or os.getenv('WATCHMAN_ENCODING') or 'bser' + transport = transport or os.getenv('WATCHMAN_TRANSPORT') or 'local' + if transport == 'local' and os.name == 'nt': + self.transport = WindowsNamedPipeTransport + elif transport == 'local': + self.transport = UnixSocketTransport + elif transport == 'cli': + self.transport = CLIProcessTransport + if sendEncoding is None: + sendEncoding = 'json' + if recvEncoding is None: + recvEncoding = sendEncoding + else: + raise WatchmanError('invalid transport %s' % transport) + + sendEncoding = str(sendEncoding or os.getenv('WATCHMAN_ENCODING') or + 'bser') + recvEncoding = str(recvEncoding or os.getenv('WATCHMAN_ENCODING') or + 'bser') self.recvCodec = self._parseEncoding(recvEncoding) self.sendCodec = self._parseEncoding(sendEncoding) @@ -581,6 +789,8 @@ if self.useImmutableBser: return ImmutableBserCodec return BserCodec + elif enc == 'experimental-bser-v2': + return Bser2WithFallbackCodec elif enc == 'json': return JsonCodec else: @@ -600,10 +810,20 @@ cmd = ['watchman', '--output-encoding=bser', 'get-sockname'] try: - p = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - close_fds=os.name != 'nt') + args = dict(stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + close_fds=os.name != 'nt') + + if os.name == 'nt': + # if invoked via an application with graphical user interface, + # this call will cause a brief command window pop-up. + # Using the flag STARTF_USESHOWWINDOW to avoid this behavior. + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + args['startupinfo'] = startupinfo + + p = subprocess.Popen(cmd, **args) + except OSError as e: raise WatchmanError('"watchman" executable not in PATH (%s)', e) @@ -614,10 +834,10 @@ raise WatchmanError("watchman exited with code %d" % exitcode) result = bser.loads(stdout) - if 'error' in result: + if b'error' in result: raise WatchmanError('get-sockname error: %s' % result['error']) - return result['sockname'] + return result[b'sockname'] def _connect(self): """ establish transport connection """ @@ -660,10 +880,16 @@ self._connect() result = self.recvConn.receive() if self._hasprop(result, 'error'): - raise CommandError(result['error']) + error = result['error'] + if compat.PYTHON3 and isinstance(self.recvConn, BserCodec): + error = result['error'].decode('utf-8', 'surrogateescape') + raise CommandError(error) if self._hasprop(result, 'log'): - self.logs.append(result['log']) + log = result['log'] + if compat.PYTHON3 and isinstance(self.recvConn, BserCodec): + log = log.decode('utf-8', 'surrogateescape') + self.logs.append(log) if self._hasprop(result, 'subscription'): sub = result['subscription'] @@ -682,6 +908,9 @@ return result def isUnilateralResponse(self, res): + if 'unilateral' in res and res['unilateral']: + return True + # Fall back to checking for known unilateral responses for k in self.unilateral: if k in res: return True @@ -712,6 +941,13 @@ remove processing impacts both the unscoped and scoped stores for the subscription data. """ + if compat.PYTHON3 and issubclass(self.recvCodec, BserCodec): + # People may pass in Unicode strings here -- but currently BSER only + # returns bytestrings. Deal with that. + if isinstance(root, str): + root = encoding.encode_local(root) + if isinstance(name, str): + name = name.encode('utf-8') if root is not None: if not root in self.sub_by_root: @@ -752,9 +988,17 @@ res = self.receive() return res - except CommandError as ex: + except EnvironmentError as ee: + # When we can depend on Python 3, we can use PEP 3134 + # exception chaining here. + raise WatchmanEnvironmentError( + 'I/O error communicating with watchman daemon', + ee.errno, + ee.strerror, + args) + except WatchmanError as ex: ex.setCommand(args) - raise ex + raise def capabilityCheck(self, optional=None, required=None): """ Perform a server capability check """ @@ -775,5 +1019,3 @@ def setTimeout(self, value): self.recvConn.setTimeout(value) self.sendConn.setTimeout(value) - -# no-check-code -- this is a 3rd party library diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/load.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/load.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/load.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/load.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,107 @@ +# Copyright 2016 Facebook, Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name Facebook nor the names of its contributors may be used to +# endorse or promote products derived from this software without specific +# prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# no unicode literals + +try: + from . import bser +except ImportError: + from . import pybser as bser + +import ctypes + +EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00" + + +def _read_bytes(fp, buf): + """Read bytes from a file-like object + + @param fp: File-like object that implements read(int) + @type fp: file + + @param buf: Buffer to read into + @type buf: bytes + + @return: buf + """ + + # Do the first read without resizing the input buffer + offset = 0 + remaining = len(buf) + while remaining > 0: + l = fp.readinto((ctypes.c_char * remaining).from_buffer(buf, offset)) + if l is None or l == 0: + return offset + offset += l + remaining -= l + return offset + + +def load(fp, mutable=True, value_encoding=None, value_errors=None): + """Deserialize a BSER-encoded blob. + + @param fp: The file-object to deserialize. + @type file: + + @param mutable: Whether to return mutable results. + @type mutable: bool + + @param value_encoding: Optional codec to use to decode values. If + unspecified or None, return values as bytestrings. + @type value_encoding: str + + @param value_errors: Optional error handler for codec. 'strict' by default. + The other most common argument is 'surrogateescape' on + Python 3. If value_encoding is None, this is ignored. + @type value_errors: str + """ + buf = ctypes.create_string_buffer(8192) + SNIFF_BUFFER_SIZE = len(EMPTY_HEADER) + header = (ctypes.c_char * SNIFF_BUFFER_SIZE).from_buffer(buf) + read_len = _read_bytes(fp, header) + if read_len < len(header): + return None + + total_len = bser.pdu_len(buf) + if total_len > len(buf): + ctypes.resize(buf, total_len) + + body = (ctypes.c_char * (total_len - len(header))).from_buffer( + buf, len(header)) + read_len = _read_bytes(fp, body) + if read_len < len(body): + raise RuntimeError('bser data ended early') + + return bser.loads( + (ctypes.c_char * total_len).from_buffer(buf, 0), + mutable, + value_encoding, + value_errors) diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/msc_stdint.h mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/msc_stdint.h --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/msc_stdint.h 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/msc_stdint.h 1970-01-01 00:00:00.000000000 +0000 @@ -1,260 +0,0 @@ -// no-check-code -// ISO C9x compliant stdint.h for Microsoft Visual Studio -// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 -// -// Copyright (c) 2006-2013 Alexander Chemeris -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// 2. Redistributions in binary form must reproduce the above copyright -// notice, this list of conditions and the following disclaimer in the -// documentation and/or other materials provided with the distribution. -// -// 3. Neither the name of the product nor the names of its contributors may -// be used to endorse or promote products derived from this software -// without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED -// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifndef _MSC_VER // [ -#error "Use this header only with Microsoft Visual C++ compilers!" -#endif // _MSC_VER ] - -#ifndef _MSC_STDINT_H_ // [ -#define _MSC_STDINT_H_ - -#if _MSC_VER > 1000 -#pragma once -#endif - -#if _MSC_VER >= 1600 // [ -#include -#else // ] _MSC_VER >= 1600 [ - -#include - -// For Visual Studio 6 in C++ mode and for many Visual Studio versions when -// compiling for ARM we should wrap include with 'extern "C++" {}' -// or compiler give many errors like this: -// error C2733: second C linkage of overloaded function 'wmemchr' not allowed -#ifdef __cplusplus -extern "C" { -#endif -# include -#ifdef __cplusplus -} -#endif - -// Define _W64 macros to mark types changing their size, like intptr_t. -#ifndef _W64 -# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 -# define _W64 __w64 -# else -# define _W64 -# endif -#endif - - -// 7.18.1 Integer types - -// 7.18.1.1 Exact-width integer types - -// Visual Studio 6 and Embedded Visual C++ 4 doesn't -// realize that, e.g. char has the same size as __int8 -// so we give up on __intX for them. -#if (_MSC_VER < 1300) - typedef signed char int8_t; - typedef signed short int16_t; - typedef signed int int32_t; - typedef unsigned char uint8_t; - typedef unsigned short uint16_t; - typedef unsigned int uint32_t; -#else - typedef signed __int8 int8_t; - typedef signed __int16 int16_t; - typedef signed __int32 int32_t; - typedef unsigned __int8 uint8_t; - typedef unsigned __int16 uint16_t; - typedef unsigned __int32 uint32_t; -#endif -typedef signed __int64 int64_t; -typedef unsigned __int64 uint64_t; - - -// 7.18.1.2 Minimum-width integer types -typedef int8_t int_least8_t; -typedef int16_t int_least16_t; -typedef int32_t int_least32_t; -typedef int64_t int_least64_t; -typedef uint8_t uint_least8_t; -typedef uint16_t uint_least16_t; -typedef uint32_t uint_least32_t; -typedef uint64_t uint_least64_t; - -// 7.18.1.3 Fastest minimum-width integer types -typedef int8_t int_fast8_t; -typedef int16_t int_fast16_t; -typedef int32_t int_fast32_t; -typedef int64_t int_fast64_t; -typedef uint8_t uint_fast8_t; -typedef uint16_t uint_fast16_t; -typedef uint32_t uint_fast32_t; -typedef uint64_t uint_fast64_t; - -// 7.18.1.4 Integer types capable of holding object pointers -#ifdef _WIN64 // [ - typedef signed __int64 intptr_t; - typedef unsigned __int64 uintptr_t; -#else // _WIN64 ][ - typedef _W64 signed int intptr_t; - typedef _W64 unsigned int uintptr_t; -#endif // _WIN64 ] - -// 7.18.1.5 Greatest-width integer types -typedef int64_t intmax_t; -typedef uint64_t uintmax_t; - - -// 7.18.2 Limits of specified-width integer types - -#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 - -// 7.18.2.1 Limits of exact-width integer types -#define INT8_MIN ((int8_t)_I8_MIN) -#define INT8_MAX _I8_MAX -#define INT16_MIN ((int16_t)_I16_MIN) -#define INT16_MAX _I16_MAX -#define INT32_MIN ((int32_t)_I32_MIN) -#define INT32_MAX _I32_MAX -#define INT64_MIN ((int64_t)_I64_MIN) -#define INT64_MAX _I64_MAX -#define UINT8_MAX _UI8_MAX -#define UINT16_MAX _UI16_MAX -#define UINT32_MAX _UI32_MAX -#define UINT64_MAX _UI64_MAX - -// 7.18.2.2 Limits of minimum-width integer types -#define INT_LEAST8_MIN INT8_MIN -#define INT_LEAST8_MAX INT8_MAX -#define INT_LEAST16_MIN INT16_MIN -#define INT_LEAST16_MAX INT16_MAX -#define INT_LEAST32_MIN INT32_MIN -#define INT_LEAST32_MAX INT32_MAX -#define INT_LEAST64_MIN INT64_MIN -#define INT_LEAST64_MAX INT64_MAX -#define UINT_LEAST8_MAX UINT8_MAX -#define UINT_LEAST16_MAX UINT16_MAX -#define UINT_LEAST32_MAX UINT32_MAX -#define UINT_LEAST64_MAX UINT64_MAX - -// 7.18.2.3 Limits of fastest minimum-width integer types -#define INT_FAST8_MIN INT8_MIN -#define INT_FAST8_MAX INT8_MAX -#define INT_FAST16_MIN INT16_MIN -#define INT_FAST16_MAX INT16_MAX -#define INT_FAST32_MIN INT32_MIN -#define INT_FAST32_MAX INT32_MAX -#define INT_FAST64_MIN INT64_MIN -#define INT_FAST64_MAX INT64_MAX -#define UINT_FAST8_MAX UINT8_MAX -#define UINT_FAST16_MAX UINT16_MAX -#define UINT_FAST32_MAX UINT32_MAX -#define UINT_FAST64_MAX UINT64_MAX - -// 7.18.2.4 Limits of integer types capable of holding object pointers -#ifdef _WIN64 // [ -# define INTPTR_MIN INT64_MIN -# define INTPTR_MAX INT64_MAX -# define UINTPTR_MAX UINT64_MAX -#else // _WIN64 ][ -# define INTPTR_MIN INT32_MIN -# define INTPTR_MAX INT32_MAX -# define UINTPTR_MAX UINT32_MAX -#endif // _WIN64 ] - -// 7.18.2.5 Limits of greatest-width integer types -#define INTMAX_MIN INT64_MIN -#define INTMAX_MAX INT64_MAX -#define UINTMAX_MAX UINT64_MAX - -// 7.18.3 Limits of other integer types - -#ifdef _WIN64 // [ -# define PTRDIFF_MIN _I64_MIN -# define PTRDIFF_MAX _I64_MAX -#else // _WIN64 ][ -# define PTRDIFF_MIN _I32_MIN -# define PTRDIFF_MAX _I32_MAX -#endif // _WIN64 ] - -#define SIG_ATOMIC_MIN INT_MIN -#define SIG_ATOMIC_MAX INT_MAX - -#ifndef SIZE_MAX // [ -# ifdef _WIN64 // [ -# define SIZE_MAX _UI64_MAX -# else // _WIN64 ][ -# define SIZE_MAX _UI32_MAX -# endif // _WIN64 ] -#endif // SIZE_MAX ] - -// WCHAR_MIN and WCHAR_MAX are also defined in -#ifndef WCHAR_MIN // [ -# define WCHAR_MIN 0 -#endif // WCHAR_MIN ] -#ifndef WCHAR_MAX // [ -# define WCHAR_MAX _UI16_MAX -#endif // WCHAR_MAX ] - -#define WINT_MIN 0 -#define WINT_MAX _UI16_MAX - -#endif // __STDC_LIMIT_MACROS ] - - -// 7.18.4 Limits of other integer types - -#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 - -// 7.18.4.1 Macros for minimum-width integer constants - -#define INT8_C(val) val##i8 -#define INT16_C(val) val##i16 -#define INT32_C(val) val##i32 -#define INT64_C(val) val##i64 - -#define UINT8_C(val) val##ui8 -#define UINT16_C(val) val##ui16 -#define UINT32_C(val) val##ui32 -#define UINT64_C(val) val##ui64 - -// 7.18.4.2 Macros for greatest-width integer constants -// These #ifndef's are needed to prevent collisions with . -// Check out Issue 9 for the details. -#ifndef INTMAX_C // [ -# define INTMAX_C INT64_C -#endif // INTMAX_C ] -#ifndef UINTMAX_C // [ -# define UINTMAX_C UINT64_C -#endif // UINTMAX_C ] - -#endif // __STDC_CONSTANT_MACROS ] - -#endif // _MSC_VER >= 1600 ] - -#endif // _MSC_STDINT_H_ ] diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/pybser.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/pybser.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/pywatchman/pybser.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/pywatchman/pybser.py 2017-01-18 16:54:51.000000000 +0000 @@ -26,33 +26,51 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +# no unicode literals + +import binascii import collections import ctypes import struct import sys -BSER_ARRAY = '\x00' -BSER_OBJECT = '\x01' -BSER_STRING = '\x02' -BSER_INT8 = '\x03' -BSER_INT16 = '\x04' -BSER_INT32 = '\x05' -BSER_INT64 = '\x06' -BSER_REAL = '\x07' -BSER_TRUE = '\x08' -BSER_FALSE = '\x09' -BSER_NULL = '\x0a' -BSER_TEMPLATE = '\x0b' -BSER_SKIP = '\x0c' +from . import ( + compat, +) + +BSER_ARRAY = b'\x00' +BSER_OBJECT = b'\x01' +BSER_BYTESTRING = b'\x02' +BSER_INT8 = b'\x03' +BSER_INT16 = b'\x04' +BSER_INT32 = b'\x05' +BSER_INT64 = b'\x06' +BSER_REAL = b'\x07' +BSER_TRUE = b'\x08' +BSER_FALSE = b'\x09' +BSER_NULL = b'\x0a' +BSER_TEMPLATE = b'\x0b' +BSER_SKIP = b'\x0c' +BSER_UTF8STRING = b'\x0d' + +if compat.PYTHON3: + STRING_TYPES = (str, bytes) + unicode = str + def tobytes(i): + return str(i).encode('ascii') + long = int +else: + STRING_TYPES = (unicode, str) + tobytes = bytes # Leave room for the serialization header, which includes # our overall length. To make things simpler, we'll use an # int32 for the header -EMPTY_HEADER = "\x00\x01\x05\x00\x00\x00\x00" - -# Python 3 conditional for supporting Python 2's int/long types -if sys.version_info > (3,): - long = int +EMPTY_HEADER = b"\x00\x01\x05\x00\x00\x00\x00" +EMPTY_HEADER_V2 = b"\x00\x02\x00\x00\x00\x00\x05\x00\x00\x00\x00" def _int_size(x): """Return the smallest size int that can store the value""" @@ -67,13 +85,28 @@ else: raise RuntimeError('Cannot represent value: ' + str(x)) +def _buf_pos(buf, pos): + ret = buf[pos] + # In Python 2, buf is a str array so buf[pos] is a string. In Python 3, buf + # is a bytes array and buf[pos] is an integer. + if compat.PYTHON3: + ret = bytes((ret,)) + return ret class _bser_buffer(object): - def __init__(self): + def __init__(self, version): + self.bser_version = version self.buf = ctypes.create_string_buffer(8192) - struct.pack_into(str(len(EMPTY_HEADER)) + 's', self.buf, 0, EMPTY_HEADER) - self.wpos = len(EMPTY_HEADER) + if self.bser_version == 1: + struct.pack_into(tobytes(len(EMPTY_HEADER)) + b's', self.buf, 0, + EMPTY_HEADER) + self.wpos = len(EMPTY_HEADER) + else: + assert self.bser_version == 2 + struct.pack_into(tobytes(len(EMPTY_HEADER_V2)) + b's', self.buf, 0, + EMPTY_HEADER_V2) + self.wpos = len(EMPTY_HEADER_V2) def ensure_size(self, size): while ctypes.sizeof(self.buf) - self.wpos < size: @@ -84,13 +117,13 @@ to_write = size + 1 self.ensure_size(to_write) if size == 1: - struct.pack_into('=cb', self.buf, self.wpos, BSER_INT8, val) + struct.pack_into(b'=cb', self.buf, self.wpos, BSER_INT8, val) elif size == 2: - struct.pack_into('=ch', self.buf, self.wpos, BSER_INT16, val) + struct.pack_into(b'=ch', self.buf, self.wpos, BSER_INT16, val) elif size == 4: - struct.pack_into('=ci', self.buf, self.wpos, BSER_INT32, val) + struct.pack_into(b'=ci', self.buf, self.wpos, BSER_INT32, val) elif size == 8: - struct.pack_into('=cq', self.buf, self.wpos, BSER_INT64, val) + struct.pack_into(b'=cq', self.buf, self.wpos, BSER_INT64, val) else: raise RuntimeError('Cannot represent this long value') self.wpos += to_write @@ -104,13 +137,17 @@ to_write = 2 + size + s_len self.ensure_size(to_write) if size == 1: - struct.pack_into('=ccb' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT8, s_len, s) + struct.pack_into(b'=ccb' + tobytes(s_len) + b's', self.buf, + self.wpos, BSER_BYTESTRING, BSER_INT8, s_len, s) elif size == 2: - struct.pack_into('=cch' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT16, s_len, s) + struct.pack_into(b'=cch' + tobytes(s_len) + b's', self.buf, + self.wpos, BSER_BYTESTRING, BSER_INT16, s_len, s) elif size == 4: - struct.pack_into('=cci' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT32, s_len, s) + struct.pack_into(b'=cci' + tobytes(s_len) + b's', self.buf, + self.wpos, BSER_BYTESTRING, BSER_INT32, s_len, s) elif size == 8: - struct.pack_into('=ccq' + str(s_len) + 's', self.buf, self.wpos, BSER_STRING, BSER_INT64, s_len, s) + struct.pack_into(b'=ccq' + tobytes(s_len) + b's', self.buf, + self.wpos, BSER_BYTESTRING, BSER_INT64, s_len, s) else: raise RuntimeError('Cannot represent this string value') self.wpos += to_write @@ -124,54 +161,68 @@ to_encode = BSER_TRUE else: to_encode = BSER_FALSE - struct.pack_into('=c', self.buf, self.wpos, to_encode) + struct.pack_into(b'=c', self.buf, self.wpos, to_encode) self.wpos += needed elif val is None: needed = 1 self.ensure_size(needed) - struct.pack_into('=c', self.buf, self.wpos, BSER_NULL) + struct.pack_into(b'=c', self.buf, self.wpos, BSER_NULL) self.wpos += needed elif isinstance(val, (int, long)): self.append_long(val) - elif isinstance(val, (str, unicode)): + elif isinstance(val, STRING_TYPES): self.append_string(val) elif isinstance(val, float): needed = 9 self.ensure_size(needed) - struct.pack_into('=cd', self.buf, self.wpos, BSER_REAL, val) + struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val) self.wpos += needed - elif isinstance(val, collections.Mapping) and isinstance(val, collections.Sized): + elif isinstance(val, collections.Mapping) and \ + isinstance(val, collections.Sized): val_len = len(val) size = _int_size(val_len) needed = 2 + size self.ensure_size(needed) if size == 1: - struct.pack_into('=ccb', self.buf, self.wpos, BSER_OBJECT, BSER_INT8, val_len) + struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_OBJECT, + BSER_INT8, val_len) elif size == 2: - struct.pack_into('=cch', self.buf, self.wpos, BSER_OBJECT, BSER_INT16, val_len) + struct.pack_into(b'=cch', self.buf, self.wpos, BSER_OBJECT, + BSER_INT16, val_len) elif size == 4: - struct.pack_into('=cci', self.buf, self.wpos, BSER_OBJECT, BSER_INT32, val_len) + struct.pack_into(b'=cci', self.buf, self.wpos, BSER_OBJECT, + BSER_INT32, val_len) elif size == 8: - struct.pack_into('=ccq', self.buf, self.wpos, BSER_OBJECT, BSER_INT64, val_len) + struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_OBJECT, + BSER_INT64, val_len) else: raise RuntimeError('Cannot represent this mapping value') self.wpos += needed - for k, v in val.iteritems(): + if compat.PYTHON3: + iteritems = val.items() + else: + iteritems = val.iteritems() + for k, v in iteritems: self.append_string(k) self.append_recursive(v) - elif isinstance(val, collections.Iterable) and isinstance(val, collections.Sized): + elif isinstance(val, collections.Iterable) and \ + isinstance(val, collections.Sized): val_len = len(val) size = _int_size(val_len) needed = 2 + size self.ensure_size(needed) if size == 1: - struct.pack_into('=ccb', self.buf, self.wpos, BSER_ARRAY, BSER_INT8, val_len) + struct.pack_into(b'=ccb', self.buf, self.wpos, BSER_ARRAY, + BSER_INT8, val_len) elif size == 2: - struct.pack_into('=cch', self.buf, self.wpos, BSER_ARRAY, BSER_INT16, val_len) + struct.pack_into(b'=cch', self.buf, self.wpos, BSER_ARRAY, + BSER_INT16, val_len) elif size == 4: - struct.pack_into('=cci', self.buf, self.wpos, BSER_ARRAY, BSER_INT32, val_len) + struct.pack_into(b'=cci', self.buf, self.wpos, BSER_ARRAY, + BSER_INT32, val_len) elif size == 8: - struct.pack_into('=ccq', self.buf, self.wpos, BSER_ARRAY, BSER_INT64, val_len) + struct.pack_into(b'=ccq', self.buf, self.wpos, BSER_ARRAY, + BSER_INT64, val_len) else: raise RuntimeError('Cannot represent this sequence value') self.wpos += needed @@ -181,56 +232,18 @@ raise RuntimeError('Cannot represent unknown value type') -def dumps(obj): - bser_buf = _bser_buffer() +def dumps(obj, version=1, capabilities=0): + bser_buf = _bser_buffer(version=version) bser_buf.append_recursive(obj) # Now fill in the overall length - obj_len = bser_buf.wpos - len(EMPTY_HEADER) - struct.pack_into('=i', bser_buf.buf, 3, obj_len) - return bser_buf.buf.raw[:bser_buf.wpos] - - -def _bunser_int(buf, pos): - try: - int_type = buf[pos] - except IndexError: - raise ValueError('Invalid bser int encoding, pos out of range') - if int_type == BSER_INT8: - needed = 2 - fmt = '=b' - elif int_type == BSER_INT16: - needed = 3 - fmt = '=h' - elif int_type == BSER_INT32: - needed = 5 - fmt = '=i' - elif int_type == BSER_INT64: - needed = 9 - fmt = '=q' + if version == 1: + obj_len = bser_buf.wpos - len(EMPTY_HEADER) + struct.pack_into(b'=i', bser_buf.buf, 3, obj_len) else: - raise ValueError('Invalid bser int encoding 0x%02x' % int(int_type)) - int_val = struct.unpack_from(fmt, buf, pos + 1)[0] - return (int_val, pos + needed) - - -def _bunser_string(buf, pos): - str_len, pos = _bunser_int(buf, pos + 1) - str_val = struct.unpack_from(str(str_len) + 's', buf, pos)[0] - return (str_val, pos + str_len) - - -def _bunser_array(buf, pos, mutable=True): - arr_len, pos = _bunser_int(buf, pos + 1) - arr = [] - for i in range(arr_len): - arr_item, pos = _bser_loads_recursive(buf, pos, mutable) - arr.append(arr_item) - - if not mutable: - arr = tuple(arr) - - return arr, pos - + obj_len = bser_buf.wpos - len(EMPTY_HEADER_V2) + struct.pack_into(b'=i', bser_buf.buf, 2, capabilities) + struct.pack_into(b'=i', bser_buf.buf, 7, obj_len) + return bser_buf.buf.raw[:bser_buf.wpos] # This is a quack-alike with the bserObjectType in bser.c # It provides by getattr accessors and getitem for both index @@ -260,100 +273,212 @@ def __len__(self): return len(self._keys) -def _bunser_object(buf, pos, mutable=True): - obj_len, pos = _bunser_int(buf, pos + 1) - if mutable: - obj = {} - else: - keys = [] - vals = [] - - for i in range(obj_len): - key, pos = _bunser_string(buf, pos) - val, pos = _bser_loads_recursive(buf, pos, mutable) - if mutable: - obj[key] = val +class Bunser(object): + def __init__(self, mutable=True, value_encoding=None, value_errors=None): + self.mutable = mutable + self.value_encoding = value_encoding + + if value_encoding is None: + self.value_errors = None + elif value_errors is None: + self.value_errors = 'strict' else: - keys.append(key) - vals.append(val) - - if not mutable: - obj = _BunserDict(keys, vals) - - return obj, pos - + self.value_errors = value_errors -def _bunser_template(buf, pos, mutable=True): - if buf[pos + 1] != BSER_ARRAY: - raise RuntimeError('Expect ARRAY to follow TEMPLATE') - keys, pos = _bunser_array(buf, pos + 1) - nitems, pos = _bunser_int(buf, pos) - arr = [] - for i in range(nitems): - if mutable: + @staticmethod + def unser_int(buf, pos): + try: + int_type = _buf_pos(buf, pos) + except IndexError: + raise ValueError('Invalid bser int encoding, pos out of range') + if int_type == BSER_INT8: + needed = 2 + fmt = b'=b' + elif int_type == BSER_INT16: + needed = 3 + fmt = b'=h' + elif int_type == BSER_INT32: + needed = 5 + fmt = b'=i' + elif int_type == BSER_INT64: + needed = 9 + fmt = b'=q' + else: + raise ValueError('Invalid bser int encoding 0x%s' % + binascii.hexlify(int_type).decode('ascii')) + int_val = struct.unpack_from(fmt, buf, pos + 1)[0] + return (int_val, pos + needed) + + def unser_utf8_string(self, buf, pos): + str_len, pos = self.unser_int(buf, pos + 1) + str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0] + return (str_val.decode('utf-8'), pos + str_len) + + def unser_bytestring(self, buf, pos): + str_len, pos = self.unser_int(buf, pos + 1) + str_val = struct.unpack_from(tobytes(str_len) + b's', buf, pos)[0] + if self.value_encoding is not None: + str_val = str_val.decode(self.value_encoding, self.value_errors) + # str_len stays the same because that's the length in bytes + return (str_val, pos + str_len) + + def unser_array(self, buf, pos): + arr_len, pos = self.unser_int(buf, pos + 1) + arr = [] + for i in range(arr_len): + arr_item, pos = self.loads_recursive(buf, pos) + arr.append(arr_item) + + if not self.mutable: + arr = tuple(arr) + + return arr, pos + + def unser_object(self, buf, pos): + obj_len, pos = self.unser_int(buf, pos + 1) + if self.mutable: obj = {} else: + keys = [] vals = [] - for keyidx in range(len(keys)): - if buf[pos] == BSER_SKIP: - pos += 1 - ele = None + for i in range(obj_len): + key, pos = self.unser_utf8_string(buf, pos) + val, pos = self.loads_recursive(buf, pos) + if self.mutable: + obj[key] = val else: - ele, pos = _bser_loads_recursive(buf, pos, mutable) + keys.append(key) + vals.append(val) - if mutable: - key = keys[keyidx] - obj[key] = ele - else: - vals.append(ele) - - if not mutable: + if not self.mutable: obj = _BunserDict(keys, vals) - arr.append(obj) - return arr, pos + return obj, pos + def unser_template(self, buf, pos): + val_type = _buf_pos(buf, pos + 1) + if val_type != BSER_ARRAY: + raise RuntimeError('Expect ARRAY to follow TEMPLATE') + # force UTF-8 on keys + keys_bunser = Bunser(mutable=self.mutable, value_encoding='utf-8') + keys, pos = keys_bunser.unser_array(buf, pos + 1) + nitems, pos = self.unser_int(buf, pos) + arr = [] + for i in range(nitems): + if self.mutable: + obj = {} + else: + vals = [] -def _bser_loads_recursive(buf, pos, mutable=True): - val_type = buf[pos] - if (val_type == BSER_INT8 or val_type == BSER_INT16 or - val_type == BSER_INT32 or val_type == BSER_INT64): - return _bunser_int(buf, pos) - elif val_type == BSER_REAL: - val = struct.unpack_from('=d', buf, pos + 1)[0] - return (val, pos + 9) - elif val_type == BSER_TRUE: - return (True, pos + 1) - elif val_type == BSER_FALSE: - return (False, pos + 1) - elif val_type == BSER_NULL: - return (None, pos + 1) - elif val_type == BSER_STRING: - return _bunser_string(buf, pos) - elif val_type == BSER_ARRAY: - return _bunser_array(buf, pos, mutable) - elif val_type == BSER_OBJECT: - return _bunser_object(buf, pos, mutable) - elif val_type == BSER_TEMPLATE: - return _bunser_template(buf, pos, mutable) + for keyidx in range(len(keys)): + if _buf_pos(buf, pos) == BSER_SKIP: + pos += 1 + ele = None + else: + ele, pos = self.loads_recursive(buf, pos) + + if self.mutable: + key = keys[keyidx] + obj[key] = ele + else: + vals.append(ele) + + if not self.mutable: + obj = _BunserDict(keys, vals) + + arr.append(obj) + return arr, pos + + def loads_recursive(self, buf, pos): + val_type = _buf_pos(buf, pos) + if (val_type == BSER_INT8 or val_type == BSER_INT16 or + val_type == BSER_INT32 or val_type == BSER_INT64): + return self.unser_int(buf, pos) + elif val_type == BSER_REAL: + val = struct.unpack_from(b'=d', buf, pos + 1)[0] + return (val, pos + 9) + elif val_type == BSER_TRUE: + return (True, pos + 1) + elif val_type == BSER_FALSE: + return (False, pos + 1) + elif val_type == BSER_NULL: + return (None, pos + 1) + elif val_type == BSER_BYTESTRING: + return self.unser_bytestring(buf, pos) + elif val_type == BSER_UTF8STRING: + return self.unser_utf8_string(buf, pos) + elif val_type == BSER_ARRAY: + return self.unser_array(buf, pos) + elif val_type == BSER_OBJECT: + return self.unser_object(buf, pos) + elif val_type == BSER_TEMPLATE: + return self.unser_template(buf, pos) + else: + raise ValueError('unhandled bser opcode 0x%s' % + binascii.hexlify(val_type).decode('ascii')) + + +def _pdu_info_helper(buf): + bser_version = -1 + if buf[0:2] == EMPTY_HEADER[0:2]: + bser_version = 1 + bser_capabilities = 0 + expected_len, pos2 = Bunser.unser_int(buf, 2) + elif buf[0:2] == EMPTY_HEADER_V2[0:2]: + if len(buf) < 8: + raise ValueError('Invalid BSER header') + bser_version = 2 + bser_capabilities = struct.unpack_from("I", buf, 2)[0] + expected_len, pos2 = Bunser.unser_int(buf, 6) else: - raise RuntimeError('unhandled bser opcode 0x%02x' % (val_type,)) + raise ValueError('Invalid BSER header') + + return bser_version, bser_capabilities, expected_len, pos2 + + +def pdu_info(buf): + info = _pdu_info_helper(buf) + return info[0], info[1], info[2] + info[3] def pdu_len(buf): - if buf[0:2] != EMPTY_HEADER[0:2]: - raise RuntimeError('Invalid BSER header') - expected_len, pos = _bunser_int(buf, 2) - return expected_len + pos + info = _pdu_info_helper(buf) + return info[2] + info[3] + + +def loads(buf, mutable=True, value_encoding=None, value_errors=None): + """Deserialize a BSER-encoded blob. + + @param buf: The buffer to deserialize. + @type buf: bytes + + @param mutable: Whether to return mutable results. + @type mutable: bool + @param value_encoding: Optional codec to use to decode values. If + unspecified or None, return values as bytestrings. + @type value_encoding: str + + @param value_errors: Optional error handler for codec. 'strict' by default. + The other most common argument is 'surrogateescape' on + Python 3. If value_encoding is None, this is ignored. + @type value_errors: str + """ + + info = _pdu_info_helper(buf) + expected_len = info[2] + pos = info[3] -def loads(buf, mutable=True): - if buf[0:2] != EMPTY_HEADER[0:2]: - raise RuntimeError('Invalid BSER header') - expected_len, pos = _bunser_int(buf, 2) if len(buf) != expected_len + pos: - raise RuntimeError('bser data len != header len') - return _bser_loads_recursive(buf, pos, mutable)[0] + raise ValueError('bser data len != header len') + + bunser = Bunser(mutable=mutable, value_encoding=value_encoding, + value_errors=value_errors) + + return bunser.loads_recursive(buf, pos)[0] + -# no-check-code -- this is a 3rd party library +def load(fp, mutable=True, value_encoding=None, value_errors=None): + from . import load + return load.load(fp, mutable, value_encoding, value_errors) diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/state.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/state.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/state.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/state.py 2017-01-18 16:54:51.000000000 +0000 @@ -59,6 +59,12 @@ state = file.read().split('\0') # state = hostname\0clock\0ignorehash\0 + list of files, each # followed by a \0 + if len(state) < 3: + self._ui.log( + 'fsmonitor', 'fsmonitor: state file truncated (expected ' + '3 chunks, found %d), nuking state\n', len(state)) + self.invalidate() + return None, None, None diskhostname = state[0] hostname = socket.gethostname() if diskhostname != hostname: @@ -85,12 +91,12 @@ return try: - file = self._opener('fsmonitor.state', 'wb') + file = self._opener('fsmonitor.state', 'wb', atomictemp=True) except (IOError, OSError): self._ui.warn(_("warning: unable to write out fsmonitor state\n")) return - try: + with file: file.write(struct.pack(_versionformat, _version)) file.write(socket.gethostname() + '\0') file.write(clock + '\0') @@ -98,8 +104,6 @@ if notefiles: file.write('\0'.join(notefiles)) file.write('\0') - finally: - file.close() def invalidate(self): try: diff -Nru mercurial-4.0.2~trusty1/hgext/fsmonitor/watchmanclient.py mercurial-4.1~rc~trusty1/hgext/fsmonitor/watchmanclient.py --- mercurial-4.0.2~trusty1/hgext/fsmonitor/watchmanclient.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/fsmonitor/watchmanclient.py 2017-01-18 16:54:51.000000000 +0000 @@ -87,7 +87,7 @@ useImmutableBser=True) return self._watchmanclient.query(*watchmanargs) except pywatchman.CommandError as ex: - if ex.msg.startswith('unable to resolve root'): + if 'unable to resolve root' in ex.msg: raise WatchmanNoRoot(self._root, ex.msg) raise Unavailable(ex.msg) except pywatchman.WatchmanError as ex: diff -Nru mercurial-4.0.2~trusty1/hgext/histedit.py mercurial-4.1~rc~trusty1/hgext/histedit.py --- mercurial-4.0.2~trusty1/hgext/histedit.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/histedit.py 2017-01-18 16:54:51.000000000 +0000 @@ -173,7 +173,6 @@ import errno import os -import sys from mercurial.i18n import _ from mercurial import ( @@ -991,9 +990,9 @@ return goaleditplan return goalnew -def _readfile(path): +def _readfile(ui, path): if path == '-': - return sys.stdin.read() + return ui.fin.read() else: with open(path, 'rb') as f: return f.read() @@ -1191,7 +1190,7 @@ node.short(state.topmost)) rules = ruleeditor(repo, ui, state.actions, comment) else: - rules = _readfile(rules) + rules = _readfile(ui, rules) actions = parserules(rules, state) ctxs = [repo[act.node] \ for act in state.actions if act.node] @@ -1232,7 +1231,7 @@ actions = [pick(state, r) for r in revs] rules = ruleeditor(repo, ui, actions, comment) else: - rules = _readfile(rules) + rules = _readfile(ui, rules) actions = parserules(rules, state) warnverifyactions(ui, repo, actions, state, ctxs) @@ -1335,7 +1334,8 @@ rules = '\n'.join([act.torule() for act in actions]) rules += '\n\n' rules += editcomment - rules = ui.edit(rules, ui.username(), {'prefix': 'histedit'}) + rules = ui.edit(rules, ui.username(), {'prefix': 'histedit'}, + tmpdir=repo.path) # Save edit rules in .hg/histedit-last-edit.txt in case # the user needs to ask for help after something @@ -1406,12 +1406,12 @@ % node.short(missing[0])) def adjustreplacementsfrommarkers(repo, oldreplacements): - """Adjust replacements from obsolescense markers + """Adjust replacements from obsolescence markers Replacements structure is originally generated based on histedit's state and does not account for changes that are not recorded there. This function fixes that by adding - data read from obsolescense markers""" + data read from obsolescence markers""" if not obsolete.isenabled(repo, obsolete.createmarkersopt): return oldreplacements diff -Nru mercurial-4.0.2~trusty1/hgext/keyword.py mercurial-4.1~rc~trusty1/hgext/keyword.py --- mercurial-4.0.2~trusty1/hgext/keyword.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/keyword.py 2017-01-18 16:54:51.000000000 +0000 @@ -737,6 +737,8 @@ return ret def kwfilectx_cmp(orig, self, fctx): + if fctx._customcmp: + return fctx.cmp(self) # keyword affects data size, comparing wdir and filelog size does # not make sense if (fctx._filenode is None and diff -Nru mercurial-4.0.2~trusty1/hgext/largefiles/lfcommands.py mercurial-4.1~rc~trusty1/hgext/largefiles/lfcommands.py --- mercurial-4.0.2~trusty1/hgext/largefiles/lfcommands.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/largefiles/lfcommands.py 2017-01-18 16:54:51.000000000 +0000 @@ -510,18 +510,21 @@ lfdirstate.normal(lfile) update1 = 1 - # copy the state of largefile standin from the repository's + # copy the exec mode of largefile standin from the repository's # dirstate to its state in the lfdirstate. rellfile = lfile relstandin = lfutil.standin(lfile) if wvfs.exists(relstandin): + # exec is decided by the users permissions using mask 0o100 standinexec = wvfs.stat(relstandin).st_mode & 0o100 - st = wvfs.stat(rellfile).st_mode - if standinexec != st & 0o100: - st &= ~0o111 + st = wvfs.stat(rellfile) + mode = st.st_mode + if standinexec != mode & 0o100: + # first remove all X bits, then shift all R bits to X + mode &= ~0o111 if standinexec: - st |= (st >> 2) & 0o111 & ~util.umask - wvfs.chmod(rellfile, st) + mode |= (mode >> 2) & 0o111 & ~util.umask + wvfs.chmod(rellfile, mode) update1 = 1 updated += update1 diff -Nru mercurial-4.0.2~trusty1/hgext/largefiles/lfutil.py mercurial-4.1~rc~trusty1/hgext/largefiles/lfutil.py --- mercurial-4.0.2~trusty1/hgext/largefiles/lfutil.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/largefiles/lfutil.py 2017-01-18 16:54:51.000000000 +0000 @@ -19,10 +19,12 @@ from mercurial import ( dirstate, + encoding, error, httpconnection, match as matchmod, node, + pycompat, scmutil, util, ) @@ -72,23 +74,25 @@ path = ui.configpath(longname, 'usercache', None) if path: return path - if os.name == 'nt': - appdata = os.getenv('LOCALAPPDATA', os.getenv('APPDATA')) + if pycompat.osname == 'nt': + appdata = encoding.environ.get('LOCALAPPDATA',\ + encoding.environ.get('APPDATA')) if appdata: return os.path.join(appdata, longname) elif platform.system() == 'Darwin': - home = os.getenv('HOME') + home = encoding.environ.get('HOME') if home: return os.path.join(home, 'Library', 'Caches', longname) - elif os.name == 'posix': - path = os.getenv('XDG_CACHE_HOME') + elif pycompat.osname == 'posix': + path = encoding.environ.get('XDG_CACHE_HOME') if path: return os.path.join(path, longname) - home = os.getenv('HOME') + home = encoding.environ.get('HOME') if home: return os.path.join(home, '.cache', longname) else: - raise error.Abort(_('unknown operating system: %s\n') % os.name) + raise error.Abort(_('unknown operating system: %s\n') + % pycompat.osname) raise error.Abort(_('unknown %s usercache location') % longname) def inusercache(ui, hash): diff -Nru mercurial-4.0.2~trusty1/hgext/largefiles/proto.py mercurial-4.1~rc~trusty1/hgext/largefiles/proto.py --- mercurial-4.0.2~trusty1/hgext/largefiles/proto.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/largefiles/proto.py 2017-01-18 16:54:51.000000000 +0000 @@ -76,7 +76,7 @@ yield '%d\n' % length for chunk in util.filechunkiter(f): yield chunk - return wireproto.streamres(generator()) + return wireproto.streamres(gen=generator()) def statlfile(repo, proto, sha): '''Server command for checking if a largefile is present - returns '2\n' if diff -Nru mercurial-4.0.2~trusty1/hgext/logtoprocess.py mercurial-4.1~rc~trusty1/hgext/logtoprocess.py --- mercurial-4.0.2~trusty1/hgext/logtoprocess.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/logtoprocess.py 2017-01-18 16:54:51.000000000 +0000 @@ -27,7 +27,7 @@ would log the warning message and traceback of any failed command dispatch. -Scripts are run asychronously as detached daemon processes; mercurial will +Scripts are run asynchronously as detached daemon processes; mercurial will not ensure that they exit cleanly. """ @@ -40,6 +40,8 @@ import subprocess import sys +from mercurial import encoding + # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for # extensions which SHIP WITH MERCURIAL. Non-mainline extensions should # be specifying the version(s) of Mercurial they are tested with, or @@ -117,7 +119,7 @@ optpairs = ( ('OPT_{0}'.format(key.upper()), str(value)) for key, value in opts.iteritems()) - env = dict(itertools.chain(os.environ.items(), + env = dict(itertools.chain(encoding.environ.items(), msgpairs, optpairs), EVENT=event, HGPID=str(os.getpid())) # Connect stdin to /dev/null to prevent child processes messing diff -Nru mercurial-4.0.2~trusty1/hgext/mq.py mercurial-4.1~rc~trusty1/hgext/mq.py --- mercurial-4.0.2~trusty1/hgext/mq.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/mq.py 2017-01-18 16:54:51.000000000 +0000 @@ -79,7 +79,7 @@ from mercurial import ( cmdutil, commands, - dispatch, + dirstateguard, error, extensions, hg, @@ -87,6 +87,7 @@ lock as lockmod, patch as patchmod, phases, + pycompat, registrar, revset, scmutil, @@ -1660,7 +1661,7 @@ # caching against the next repo.status call mm, aa, dd = repo.status(patchparent, top)[:3] changes = repo.changelog.read(top) - man = repo.manifest.read(changes[0]) + man = repo.manifestlog[changes[0]].read() aaa = aa[:] matchfn = scmutil.match(repo[None], pats, opts) # in short mode, we only diff the files included in the @@ -1725,7 +1726,7 @@ dsguard = None try: - dsguard = cmdutil.dirstateguard(repo, 'mq.refresh') + dsguard = dirstateguard.dirstateguard(repo, 'mq.refresh') if diffopts.git or diffopts.upgrade: copies = {} for dst in a: @@ -3523,7 +3524,7 @@ raise error.Abort(_('only a local queue repository ' 'may be initialized')) else: - repopath = cmdutil.findrepo(os.getcwd()) + repopath = cmdutil.findrepo(pycompat.getcwd()) if not repopath: raise error.Abort(_('there is no Mercurial repository here ' '(.hg not found)')) @@ -3588,7 +3589,7 @@ for cmd, entry in cmdtable.iteritems(): cmd = cmdutil.parsealiases(cmd)[0] func = entry[0] - if dispatch._cmdattr(ui, cmd, func, 'norepo'): + if func.norepo: continue entry = extensions.wrapcommand(cmdtable, cmd, mqcommand) entry[1].extend(mqopt) diff -Nru mercurial-4.0.2~trusty1/hgext/pager.py mercurial-4.1~rc~trusty1/hgext/pager.py --- mercurial-4.0.2~trusty1/hgext/pager.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/pager.py 2017-01-18 16:54:51.000000000 +0000 @@ -71,6 +71,7 @@ cmdutil, commands, dispatch, + encoding, extensions, util, ) @@ -84,19 +85,19 @@ def _runpager(ui, p): pager = subprocess.Popen(p, shell=True, bufsize=-1, close_fds=util.closefds, stdin=subprocess.PIPE, - stdout=sys.stdout, stderr=sys.stderr) + stdout=util.stdout, stderr=util.stderr) # back up original file objects and descriptors olduifout = ui.fout - oldstdout = sys.stdout - stdoutfd = os.dup(sys.stdout.fileno()) - stderrfd = os.dup(sys.stderr.fileno()) + oldstdout = util.stdout + stdoutfd = os.dup(util.stdout.fileno()) + stderrfd = os.dup(util.stderr.fileno()) # create new line-buffered stdout so that output can show up immediately - ui.fout = sys.stdout = newstdout = os.fdopen(sys.stdout.fileno(), 'wb', 1) - os.dup2(pager.stdin.fileno(), sys.stdout.fileno()) - if ui._isatty(sys.stderr): - os.dup2(pager.stdin.fileno(), sys.stderr.fileno()) + ui.fout = util.stdout = newstdout = os.fdopen(util.stdout.fileno(), 'wb', 1) + os.dup2(pager.stdin.fileno(), util.stdout.fileno()) + if ui._isatty(util.stderr): + os.dup2(pager.stdin.fileno(), util.stderr.fileno()) @atexit.register def killpager(): @@ -104,26 +105,27 @@ signal.signal(signal.SIGINT, signal.SIG_IGN) pager.stdin.close() ui.fout = olduifout - sys.stdout = oldstdout + util.stdout = oldstdout # close new stdout while it's associated with pager; otherwise stdout # fd would be closed when newstdout is deleted newstdout.close() # restore original fds: stdout is open again - os.dup2(stdoutfd, sys.stdout.fileno()) - os.dup2(stderrfd, sys.stderr.fileno()) + os.dup2(stdoutfd, util.stdout.fileno()) + os.dup2(stderrfd, util.stderr.fileno()) pager.wait() def uisetup(ui): if '--debugger' in sys.argv or not ui.formatted(): return - # chg has its own pager implementation - argv = sys.argv[:] - if 'chgunix' in dispatch._earlygetopt(['--cmdserver'], argv): - return + class pagerui(ui.__class__): + def _runpager(self, pagercmd): + _runpager(self, pagercmd) + + ui.__class__ = pagerui def pagecmd(orig, ui, options, cmd, cmdfunc): - p = ui.config("pager", "pager", os.environ.get("PAGER")) + p = ui.config("pager", "pager", encoding.environ.get("PAGER")) usepager = False always = util.parsebool(options['pager']) auto = options['pager'] == 'auto' @@ -156,7 +158,7 @@ ui.setconfig('ui', 'interactive', False, 'pager') if util.safehasattr(signal, "SIGPIPE"): signal.signal(signal.SIGPIPE, signal.SIG_DFL) - _runpager(ui, p) + ui._runpager(p) return orig(ui, options, cmd, cmdfunc) # Wrap dispatch._runcommand after color is loaded so color can see diff -Nru mercurial-4.0.2~trusty1/hgext/patchbomb.py mercurial-4.1~rc~trusty1/hgext/patchbomb.py --- mercurial-4.0.2~trusty1/hgext/patchbomb.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/patchbomb.py 2017-01-18 16:54:51.000000000 +0000 @@ -75,6 +75,7 @@ from mercurial import ( cmdutil, commands, + encoding, error, hg, mail, @@ -166,7 +167,7 @@ while patchlines and not patchlines[0].strip(): patchlines.pop(0) - ds = patch.diffstat(patchlines, git=opts.get('git')) + ds = patch.diffstat(patchlines) if opts.get('diffstat'): body += ds + '\n\n' @@ -270,7 +271,7 @@ else: ui.write(_('\nWrite the introductory message for the ' 'patch series.\n\n')) - body = ui.edit(defaultbody, sender) + body = ui.edit(defaultbody, sender, tmpdir=repo.path) # Save series description in case sendmail fails msgfile = repo.vfs('last-email.txt', 'wb') msgfile.write(body) @@ -693,8 +694,8 @@ if opts.get('test'): ui.status(_('displaying '), subj, ' ...\n') ui.flush() - if 'PAGER' in os.environ and not ui.plain(): - fp = util.popen(os.environ['PAGER'], 'w') + if 'PAGER' in encoding.environ and not ui.plain(): + fp = util.popen(encoding.environ['PAGER'], 'w') else: fp = ui generator = emailmod.Generator.Generator(fp, mangle_from_=False) diff -Nru mercurial-4.0.2~trusty1/hgext/rebase.py mercurial-4.1~rc~trusty1/hgext/rebase.py --- mercurial-4.0.2~trusty1/hgext/rebase.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/rebase.py 2017-01-18 16:54:51.000000000 +0000 @@ -32,11 +32,13 @@ commands, copies, destutil, + dirstateguard, error, extensions, hg, lock, - merge, + merge as mergemod, + mergeutil, obsolete, patch, phases, @@ -482,19 +484,20 @@ ui.note(_("update back to initial working directory parent\n")) hg.updaterepo(repo, newwd, False) + if self.currentbookmarks: + with repo.transaction('bookmark') as tr: + updatebookmarks(repo, targetnode, nstate, + self.currentbookmarks, tr) + if self.activebookmark not in repo._bookmarks: + # active bookmark was divergent one and has been deleted + self.activebookmark = None + if not self.keepf: collapsedas = None if self.collapsef: collapsedas = newnode clearrebased(ui, repo, self.state, self.skipped, collapsedas) - with repo.transaction('bookmark') as tr: - if self.currentbookmarks: - updatebookmarks(repo, targetnode, nstate, - self.currentbookmarks, tr) - if self.activebookmark not in repo._bookmarks: - # active bookmark was divergent one and has been deleted - self.activebookmark = None clearstatus(repo) clearcollapsemsg(repo) @@ -661,6 +664,9 @@ _('abort and continue do not allow specifying revisions')) if abortf and opts.get('tool', False): ui.warn(_('tool option will be ignored\n')) + if contf: + ms = mergemod.mergestate.read(repo) + mergeutil.checkunresolved(ms) retcode = rbsrt._prepareabortorcontinue(abortf) if retcode is not None: @@ -718,12 +724,19 @@ dest = repo[_destrebase(repo, base, destspace=destspace)] destf = str(dest) - commonanc = repo.revs('ancestor(%ld, %d)', base, dest).first() - if commonanc is not None: - rebaseset = repo.revs('(%d::(%ld) - %d)::', - commonanc, base, commonanc) - else: - rebaseset = [] + roots = [] # selected children of branching points + bpbase = {} # {branchingpoint: [origbase]} + for b in base: # group bases by branching points + bp = repo.revs('ancestor(%d, %d)', b, dest).first() + bpbase[bp] = bpbase.get(bp, []) + [b] + if None in bpbase: + # emulate the old behavior, showing "nothing to rebase" (a better + # behavior may be abort with "cannot find branching point" error) + bpbase.clear() + for bp, bs in bpbase.iteritems(): # calculate roots + roots += list(repo.revs('children(%d) & ancestors(%ld)', bp, bs)) + + rebaseset = repo.revs('%ld::', roots) if not rebaseset: # transform to list because smartsets are not comparable to @@ -786,7 +799,7 @@ '''Commit the wd changes with parents p1 and p2. Reuse commit info from rev but also store useful information in extra. Return node of committed revision.''' - dsguard = cmdutil.dirstateguard(repo, 'rebase') + dsguard = dirstateguard.dirstateguard(repo, 'rebase') try: repo.setparents(repo[p1].node(), repo[p2].node()) ctx = repo[rev] @@ -823,7 +836,7 @@ # Update to target and merge it with local if repo['.'].rev() != p1: repo.ui.debug(" update to %d:%s\n" % (p1, repo[p1])) - merge.update(repo, p1, False, True) + mergemod.update(repo, p1, False, True) else: repo.ui.debug(" already in target\n") repo.dirstate.write(repo.currenttransaction()) @@ -832,8 +845,8 @@ repo.ui.debug(" detach base %d:%s\n" % (base, repo[base])) # When collapsing in-place, the parent is the common ancestor, we # have to allow merging with it. - stats = merge.update(repo, rev, True, True, base, collapse, - labels=['dest', 'source']) + stats = mergemod.update(repo, rev, True, True, base, collapse, + labels=['dest', 'source']) if collapse: copies.duplicatecopies(repo, rev, target) else: @@ -855,10 +868,7 @@ else: return None -def _checkobsrebase(repo, ui, - rebaseobsrevs, - rebasesetrevs, - rebaseobsskipped): +def _checkobsrebase(repo, ui, rebaseobsrevs, rebasesetrevs, rebaseobsskipped): """ Abort if rebase will create divergence or rebase is noop because of markers @@ -1150,7 +1160,7 @@ # Update away from the rebase if necessary if shouldupdate or needupdate(repo, state): - merge.update(repo, originalwd, False, True) + mergemod.update(repo, originalwd, False, True) # Strip from the first rebased revision if rebased: @@ -1306,6 +1316,10 @@ ui.debug('--update and --rebase are not compatible, ignoring ' 'the update flag\n') + cmdutil.checkunfinished(repo) + cmdutil.bailifchanged(repo, hint=_('cannot pull with rebase: ' + 'please commit or shelve your changes first')) + revsprepull = len(repo) origpostincoming = commands.postincoming def _dummy(*args, **kwargs): @@ -1382,7 +1396,7 @@ """return a mapping obsolete => successor for all obsolete nodes to be rebased that have a successors in the destination - obsolete => None entries in the mapping indicate nodes with no succesor""" + obsolete => None entries in the mapping indicate nodes with no successor""" obsoletenotrebased = {} # Build a mapping successor => obsolete nodes for the obsolete @@ -1414,7 +1428,7 @@ return obsoletenotrebased def summaryhook(ui, repo): - if not os.path.exists(repo.join('rebasestate')): + if not repo.vfs.exists('rebasestate'): return try: rbsrt = rebaseruntime(repo, ui, {}) diff -Nru mercurial-4.0.2~trusty1/hgext/schemes.py mercurial-4.1~rc~trusty1/hgext/schemes.py --- mercurial-4.0.2~trusty1/hgext/schemes.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/schemes.py 2017-01-18 16:54:51.000000000 +0000 @@ -50,6 +50,7 @@ error, extensions, hg, + pycompat, templater, util, ) @@ -114,7 +115,7 @@ schemes.update(dict(ui.configitems('schemes'))) t = templater.engine(lambda x: x) for scheme, url in schemes.items(): - if (os.name == 'nt' and len(scheme) == 1 and scheme.isalpha() + if (pycompat.osname == 'nt' and len(scheme) == 1 and scheme.isalpha() and os.path.exists('%s:\\' % scheme)): raise error.Abort(_('custom scheme %s:// conflicts with drive ' 'letter %s:\\\n') % (scheme, scheme.upper())) diff -Nru mercurial-4.0.2~trusty1/hgext/shelve.py mercurial-4.1~rc~trusty1/hgext/shelve.py --- mercurial-4.0.2~trusty1/hgext/shelve.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/shelve.py 2017-01-18 16:54:51.000000000 +0000 @@ -62,6 +62,13 @@ backupdir = 'shelve-backup' shelvedir = 'shelved' +shelvefileextensions = ['hg', 'patch'] +# universal extension is present in all types of shelves +patchextension = 'patch' + +# we never need the user, so we use a +# generic user for all shelve operations +shelveuser = 'shelve@localhost' class shelvedfile(object): """Helper for the file storing a single shelve @@ -154,6 +161,8 @@ """ _version = 1 _filename = 'shelvedstate' + _keep = 'keep' + _nokeep = 'nokeep' @classmethod def load(cls, repo): @@ -170,6 +179,7 @@ parents = [nodemod.bin(h) for h in fp.readline().split()] stripnodes = [nodemod.bin(h) for h in fp.readline().split()] branchtorestore = fp.readline().strip() + keep = fp.readline().strip() == cls._keep except (ValueError, TypeError) as err: raise error.CorruptedState(str(err)) finally: @@ -183,6 +193,7 @@ obj.parents = parents obj.stripnodes = stripnodes obj.branchtorestore = branchtorestore + obj.keep = keep except error.RepoLookupError as err: raise error.CorruptedState(str(err)) @@ -190,7 +201,7 @@ @classmethod def save(cls, repo, name, originalwctx, pendingctx, stripnodes, - branchtorestore): + branchtorestore, keep=False): fp = repo.vfs(cls._filename, 'wb') fp.write('%i\n' % cls._version) fp.write('%s\n' % name) @@ -201,6 +212,7 @@ fp.write('%s\n' % ' '.join([nodemod.hex(n) for n in stripnodes])) fp.write('%s\n' % branchtorestore) + fp.write('%s\n' % (cls._keep if keep else cls._nokeep)) fp.close() @classmethod @@ -210,7 +222,8 @@ def cleanupoldbackups(repo): vfs = scmutil.vfs(repo.join(backupdir)) maxbackups = repo.ui.configint('shelve', 'maxbackups', 10) - hgfiles = [f for f in vfs.listdir() if f.endswith('.hg')] + hgfiles = [f for f in vfs.listdir() + if f.endswith('.' + patchextension)] hgfiles = sorted([(vfs.stat(f).st_mtime, f) for f in hgfiles]) if 0 < maxbackups and maxbackups < len(hgfiles): bordermtime = hgfiles[-maxbackups][0] @@ -220,8 +233,8 @@ if mtime == bordermtime: # keep it, because timestamp can't decide exact order of backups continue - base = f[:-3] - for ext in 'hg patch'.split(): + base = f[:-(1 + len(patchextension))] + for ext in shelvefileextensions: try: vfs.unlink(base + '.' + ext) except OSError as err: @@ -242,24 +255,111 @@ cmdutil.checkunfinished(repo) return _docreatecmd(ui, repo, pats, opts) -def _docreatecmd(ui, repo, pats, opts): - def mutableancestors(ctx): - """return all mutable ancestors for ctx (included) +def getshelvename(repo, parent, opts): + """Decide on the name this shelve is going to have""" + def gennames(): + yield label + for i in xrange(1, 100): + yield '%s-%02d' % (label, i) + name = opts.get('name') + label = repo._activebookmark or parent.branch() or 'default' + # slashes aren't allowed in filenames, therefore we rename it + label = label.replace('/', '_') + label = label.replace('\\', '_') + # filenames must not start with '.' as it should not be hidden + if label.startswith('.'): + label = label.replace('.', '_', 1) + + if name: + if shelvedfile(repo, name, patchextension).exists(): + e = _("a shelved change named '%s' already exists") % name + raise error.Abort(e) + + # ensure we are not creating a subdirectory or a hidden file + if '/' in name or '\\' in name: + raise error.Abort(_('shelved change names can not contain slashes')) + if name.startswith('.'): + raise error.Abort(_("shelved change names can not start with '.'")) + + else: + for n in gennames(): + if not shelvedfile(repo, n, patchextension).exists(): + name = n + break + else: + raise error.Abort(_("too many shelved changes named '%s'") % label) + + return name - Much faster than the revset ancestors(ctx) & draft()""" - seen = set([nodemod.nullrev]) - visit = collections.deque() - visit.append(ctx) - while visit: - ctx = visit.popleft() - yield ctx.node() - for parent in ctx.parents(): - rev = parent.rev() - if rev not in seen: - seen.add(rev) - if parent.mutable(): - visit.append(parent) +def mutableancestors(ctx): + """return all mutable ancestors for ctx (included) + Much faster than the revset ancestors(ctx) & draft()""" + seen = set([nodemod.nullrev]) + visit = collections.deque() + visit.append(ctx) + while visit: + ctx = visit.popleft() + yield ctx.node() + for parent in ctx.parents(): + rev = parent.rev() + if rev not in seen: + seen.add(rev) + if parent.mutable(): + visit.append(parent) + +def getcommitfunc(extra, interactive, editor=False): + def commitfunc(ui, repo, message, match, opts): + hasmq = util.safehasattr(repo, 'mq') + if hasmq: + saved, repo.mq.checkapplied = repo.mq.checkapplied, False + backup = repo.ui.backupconfig('phases', 'new-commit') + try: + repo.ui.setconfig('phases', 'new-commit', phases.secret) + editor_ = False + if editor: + editor_ = cmdutil.getcommiteditor(editform='shelve.shelve', + **opts) + return repo.commit(message, shelveuser, opts.get('date'), match, + editor=editor_, extra=extra) + finally: + repo.ui.restoreconfig(backup) + if hasmq: + repo.mq.checkapplied = saved + + def interactivecommitfunc(ui, repo, *pats, **opts): + match = scmutil.match(repo['.'], pats, {}) + message = opts['message'] + return commitfunc(ui, repo, message, match, opts) + + return interactivecommitfunc if interactive else commitfunc + +def _nothingtoshelvemessaging(ui, repo, pats, opts): + stat = repo.status(match=scmutil.match(repo[None], pats, opts)) + if stat.deleted: + ui.status(_("nothing changed (%d missing files, see " + "'hg status')\n") % len(stat.deleted)) + else: + ui.status(_("nothing changed\n")) + +def _shelvecreatedcommit(repo, node, name): + bases = list(mutableancestors(repo[node])) + shelvedfile(repo, name, 'hg').writebundle(bases, node) + cmdutil.export(repo, [node], + fp=shelvedfile(repo, name, patchextension).opener('wb'), + opts=mdiff.diffopts(git=True)) + +def _includeunknownfiles(repo, pats, opts, extra): + s = repo.status(match=scmutil.match(repo[None], pats, opts), + unknown=True) + if s.unknown: + extra['shelve_unknown'] = '\0'.join(s.unknown) + repo[None].add(s.unknown) + +def _finishshelve(repo): + _aborttransaction(repo) + +def _docreatecmd(ui, repo, pats, opts): wctx = repo[None] parents = wctx.parents() if len(parents) > 1: @@ -267,18 +367,6 @@ parent = parents[0] origbranch = wctx.branch() - # we never need the user, so we use a generic user for all shelve operations - user = 'shelve@localhost' - label = repo._activebookmark or parent.branch() or 'default' - - # slashes aren't allowed in filenames, therefore we rename it - label = label.replace('/', '_') - - def gennames(): - yield label - for i in xrange(1, 100): - yield '%s-%02d' % (label, i) - if parent.node() != nodemod.nullid: desc = "changes to: %s" % parent.description().split('\n', 1)[0] else: @@ -287,8 +375,6 @@ if not opts.get('message'): opts['message'] = desc - name = opts.get('name') - lock = tr = None try: lock = repo.lock() @@ -297,81 +383,31 @@ # pull races. ensure we don't print the abort message to stderr. tr = repo.transaction('commit', report=lambda x: None) - if name: - if shelvedfile(repo, name, 'hg').exists(): - raise error.Abort(_("a shelved change named '%s' already exists" - ) % name) - else: - for n in gennames(): - if not shelvedfile(repo, n, 'hg').exists(): - name = n - break - else: - raise error.Abort(_("too many shelved changes named '%s'") % - label) - - # ensure we are not creating a subdirectory or a hidden file - if '/' in name or '\\' in name: - raise error.Abort(_('shelved change names may not contain slashes')) - if name.startswith('.'): - raise error.Abort(_("shelved change names may not start with '.'")) interactive = opts.get('interactive', False) includeunknown = (opts.get('unknown', False) and not opts.get('addremove', False)) - extra={} + name = getshelvename(repo, parent, opts) + extra = {} if includeunknown: - s = repo.status(match=scmutil.match(repo[None], pats, opts), - unknown=True) - if s.unknown: - extra['shelve_unknown'] = '\0'.join(s.unknown) - repo[None].add(s.unknown) + _includeunknownfiles(repo, pats, opts, extra) if _iswctxonnewbranch(repo) and not _isbareshelve(pats, opts): # In non-bare shelve we don't store newly created branch # at bundled commit repo.dirstate.setbranch(repo['.'].branch()) - def commitfunc(ui, repo, message, match, opts): - hasmq = util.safehasattr(repo, 'mq') - if hasmq: - saved, repo.mq.checkapplied = repo.mq.checkapplied, False - backup = repo.ui.backupconfig('phases', 'new-commit') - try: - repo.ui. setconfig('phases', 'new-commit', phases.secret) - editor = cmdutil.getcommiteditor(editform='shelve.shelve', - **opts) - return repo.commit(message, user, opts.get('date'), match, - editor=editor, extra=extra) - finally: - repo.ui.restoreconfig(backup) - if hasmq: - repo.mq.checkapplied = saved - - def interactivecommitfunc(ui, repo, *pats, **opts): - match = scmutil.match(repo['.'], pats, {}) - message = opts['message'] - return commitfunc(ui, repo, message, match, opts) + commitfunc = getcommitfunc(extra, interactive, editor=True) if not interactive: node = cmdutil.commit(ui, repo, commitfunc, pats, opts) else: - node = cmdutil.dorecord(ui, repo, interactivecommitfunc, None, + node = cmdutil.dorecord(ui, repo, commitfunc, None, False, cmdutil.recordfilter, *pats, **opts) if not node: - stat = repo.status(match=scmutil.match(repo[None], pats, opts)) - if stat.deleted: - ui.status(_("nothing changed (%d missing files, see " - "'hg status')\n") % len(stat.deleted)) - else: - ui.status(_("nothing changed\n")) + _nothingtoshelvemessaging(ui, repo, pats, opts) return 1 - bases = list(mutableancestors(repo[node])) - shelvedfile(repo, name, 'hg').writebundle(bases, node) - cmdutil.export(repo, [node], - fp=shelvedfile(repo, name, 'patch').opener('wb'), - opts=mdiff.diffopts(git=True)) - + _shelvecreatedcommit(repo, node, name) if ui.formatted(): desc = util.ellipsis(desc, ui.termwidth()) @@ -380,7 +416,7 @@ if origbranch != repo['.'].branch() and not _isbareshelve(pats, opts): repo.dirstate.setbranch(origbranch) - _aborttransaction(repo) + _finishshelve(repo) finally: lockmod.release(tr, lock) @@ -399,7 +435,7 @@ with repo.wlock(): for (name, _type) in repo.vfs.readdir(shelvedir): suffix = name.rsplit('.', 1)[-1] - if suffix in ('hg', 'patch'): + if suffix in shelvefileextensions: shelvedfile(repo, name).movetobackup() cleanupoldbackups(repo) @@ -410,8 +446,15 @@ with repo.wlock(): try: for name in pats: - for suffix in 'hg patch'.split(): - shelvedfile(repo, name, suffix).movetobackup() + for suffix in shelvefileextensions: + shfile = shelvedfile(repo, name, suffix) + # patch file is necessary, as it should + # be present for any kind of shelve, + # but the .hg file is optional as in future we + # will add obsolete shelve with does not create a + # bundle + if shfile.exists() or suffix == patchextension: + shfile.movetobackup() cleanupoldbackups(repo) except OSError as err: if err.errno != errno.ENOENT: @@ -429,7 +472,7 @@ info = [] for (name, _type) in names: pfx, sfx = name.rsplit('.', 1) - if not pfx or sfx != 'patch': + if not pfx or sfx != patchextension: continue st = shelvedfile(repo, name).stat() info.append((st.st_mtime, shelvedfile(repo, pfx).filename())) @@ -457,7 +500,7 @@ ui.write(age, label='shelve.age') ui.write(' ' * (12 - len(age))) used += 12 - with open(name + '.patch', 'rb') as fp: + with open(name + '.' + patchextension, 'rb') as fp: while True: line = fp.readline() if not line: @@ -476,18 +519,17 @@ for chunk, label in patch.difflabel(iter, difflines): ui.write(chunk, label=label) if opts['stat']: - for chunk, label in patch.diffstatui(difflines, width=width, - git=True): + for chunk, label in patch.diffstatui(difflines, width=width): ui.write(chunk, label=label) -def singlepatchcmds(ui, repo, pats, opts, subcommand): - """subcommand that displays a single shelf""" - if len(pats) != 1: - raise error.Abort(_("--%s expects a single shelf") % subcommand) - shelfname = pats[0] - - if not shelvedfile(repo, shelfname, 'patch').exists(): - raise error.Abort(_("cannot find shelf %s") % shelfname) +def patchcmds(ui, repo, pats, opts, subcommand): + """subcommand that displays shelves""" + if len(pats) == 0: + raise error.Abort(_("--%s expects at least one shelf") % subcommand) + + for shelfname in pats: + if not shelvedfile(repo, shelfname, patchextension).exists(): + raise error.Abort(_("cannot find shelf %s") % shelfname) listcmd(ui, repo, pats, opts) @@ -557,8 +599,10 @@ def unshelvecleanup(ui, repo, name, opts): """remove related files after an unshelve""" if not opts.get('keep'): - for filetype in 'hg patch'.split(): - shelvedfile(repo, name, filetype).movetobackup() + for filetype in shelvefileextensions: + shfile = shelvedfile(repo, name, filetype) + if shfile.exists(): + shfile.movetobackup() cleanupoldbackups(repo) def unshelvecontinue(ui, repo, state, opts): @@ -600,6 +644,91 @@ unshelvecleanup(ui, repo, state.name, opts) ui.status(_("unshelve of '%s' complete\n") % state.name) +def _commitworkingcopychanges(ui, repo, opts, tmpwctx): + """Temporarily commit working copy changes before moving unshelve commit""" + # Store pending changes in a commit and remember added in case a shelve + # contains unknown files that are part of the pending change + s = repo.status() + addedbefore = frozenset(s.added) + if not (s.modified or s.added or s.removed or s.deleted): + return tmpwctx, addedbefore + ui.status(_("temporarily committing pending changes " + "(restore with 'hg unshelve --abort')\n")) + commitfunc = getcommitfunc(extra=None, interactive=False, + editor=False) + tempopts = {} + tempopts['message'] = "pending changes temporary commit" + tempopts['date'] = opts.get('date') + ui.quiet = True + node = cmdutil.commit(ui, repo, commitfunc, [], tempopts) + tmpwctx = repo[node] + return tmpwctx, addedbefore + +def _unshelverestorecommit(ui, repo, basename, oldquiet): + """Recreate commit in the repository during the unshelve""" + ui.quiet = True + shelvedfile(repo, basename, 'hg').applybundle() + shelvectx = repo['tip'] + ui.quiet = oldquiet + return repo, shelvectx + +def _rebaserestoredcommit(ui, repo, opts, tr, oldtiprev, basename, pctx, + tmpwctx, shelvectx, branchtorestore): + """Rebase restored commit from its original location to a destination""" + # If the shelve is not immediately on top of the commit + # we'll be merging with, rebase it to be on top. + if tmpwctx.node() == shelvectx.parents()[0].node(): + return shelvectx + + ui.status(_('rebasing shelved changes\n')) + try: + rebase.rebase(ui, repo, **{ + 'rev': [shelvectx.rev()], + 'dest': str(tmpwctx.rev()), + 'keep': True, + 'tool': opts.get('tool', ''), + }) + except error.InterventionRequired: + tr.close() + + stripnodes = [repo.changelog.node(rev) + for rev in xrange(oldtiprev, len(repo))] + shelvedstate.save(repo, basename, pctx, tmpwctx, stripnodes, + branchtorestore, opts.get('keep')) + + util.rename(repo.join('rebasestate'), + repo.join('unshelverebasestate')) + raise error.InterventionRequired( + _("unresolved conflicts (see 'hg resolve', then " + "'hg unshelve --continue')")) + + # refresh ctx after rebase completes + shelvectx = repo['tip'] + + if not shelvectx in tmpwctx.children(): + # rebase was a no-op, so it produced no child commit + shelvectx = tmpwctx + return shelvectx + +def _forgetunknownfiles(repo, shelvectx, addedbefore): + # Forget any files that were unknown before the shelve, unknown before + # unshelve started, but are now added. + shelveunknown = shelvectx.extra().get('shelve_unknown') + if not shelveunknown: + return + shelveunknown = frozenset(shelveunknown.split('\0')) + addedafter = frozenset(repo.status().added) + toforget = (addedafter & shelveunknown) - addedbefore + repo[None].forget(toforget) + +def _finishunshelve(repo, oldtiprev, tr): + # The transaction aborting will strip all the commits for us, + # but it doesn't update the inmemory structures, so addchangegroup + # hooks still fire and try to operate on the missing commits. + # Clean up manually to prevent this. + repo.unfiltered().changelog.strip(oldtiprev, tr) + _aborttransaction(repo) + @command('unshelve', [('a', 'abort', None, _('abort an incomplete unshelve operation')), @@ -667,6 +796,8 @@ try: state = shelvedstate.load(repo) + if opts.get('keep') is None: + opts['keep'] = state.keep except IOError as err: if err.errno != errno.ENOENT: raise @@ -701,7 +832,7 @@ else: basename = shelved[0] - if not shelvedfile(repo, basename, 'patch').exists(): + if not shelvedfile(repo, basename, patchextension).exists(): raise error.Abort(_("shelved change '%s' not found") % basename) oldquiet = ui.quiet @@ -722,101 +853,25 @@ # and shelvectx is the unshelved changes. Then we merge it all down # to the original pctx. - # Store pending changes in a commit and remember added in case a shelve - # contains unknown files that are part of the pending change - s = repo.status() - addedbefore = frozenset(s.added) - if s.modified or s.added or s.removed or s.deleted: - ui.status(_("temporarily committing pending changes " - "(restore with 'hg unshelve --abort')\n")) - def commitfunc(ui, repo, message, match, opts): - hasmq = util.safehasattr(repo, 'mq') - if hasmq: - saved, repo.mq.checkapplied = repo.mq.checkapplied, False - - backup = repo.ui.backupconfig('phases', 'new-commit') - try: - repo.ui.setconfig('phases', 'new-commit', phases.secret) - return repo.commit(message, 'shelve@localhost', - opts.get('date'), match) - finally: - repo.ui.restoreconfig(backup) - if hasmq: - repo.mq.checkapplied = saved - - tempopts = {} - tempopts['message'] = "pending changes temporary commit" - tempopts['date'] = opts.get('date') - ui.quiet = True - node = cmdutil.commit(ui, repo, commitfunc, [], tempopts) - tmpwctx = repo[node] + tmpwctx, addedbefore = _commitworkingcopychanges(ui, repo, opts, + tmpwctx) - ui.quiet = True - shelvedfile(repo, basename, 'hg').applybundle() - - ui.quiet = oldquiet - - shelvectx = repo['tip'] + repo, shelvectx = _unshelverestorecommit(ui, repo, basename, oldquiet) branchtorestore = '' if shelvectx.branch() != shelvectx.p1().branch(): branchtorestore = shelvectx.branch() - # If the shelve is not immediately on top of the commit - # we'll be merging with, rebase it to be on top. - if tmpwctx.node() != shelvectx.parents()[0].node(): - ui.status(_('rebasing shelved changes\n')) - try: - rebase.rebase(ui, repo, **{ - 'rev' : [shelvectx.rev()], - 'dest' : str(tmpwctx.rev()), - 'keep' : True, - 'tool' : opts.get('tool', ''), - }) - except error.InterventionRequired: - tr.close() - - stripnodes = [repo.changelog.node(rev) - for rev in xrange(oldtiprev, len(repo))] - shelvedstate.save(repo, basename, pctx, tmpwctx, stripnodes, - branchtorestore) - - util.rename(repo.join('rebasestate'), - repo.join('unshelverebasestate')) - raise error.InterventionRequired( - _("unresolved conflicts (see 'hg resolve', then " - "'hg unshelve --continue')")) - - # refresh ctx after rebase completes - shelvectx = repo['tip'] - - if not shelvectx in tmpwctx.children(): - # rebase was a no-op, so it produced no child commit - shelvectx = tmpwctx - + shelvectx = _rebaserestoredcommit(ui, repo, opts, tr, oldtiprev, + basename, pctx, tmpwctx, shelvectx, + branchtorestore) mergefiles(ui, repo, pctx, shelvectx) restorebranch(ui, repo, branchtorestore) - - # Forget any files that were unknown before the shelve, unknown before - # unshelve started, but are now added. - shelveunknown = shelvectx.extra().get('shelve_unknown') - if shelveunknown: - shelveunknown = frozenset(shelveunknown.split('\0')) - addedafter = frozenset(repo.status().added) - toforget = (addedafter & shelveunknown) - addedbefore - repo[None].forget(toforget) + _forgetunknownfiles(repo, shelvectx, addedbefore) shelvedstate.clear(repo) - - # The transaction aborting will strip all the commits for us, - # but it doesn't update the inmemory structures, so addchangegroup - # hooks still fire and try to operate on the missing commits. - # Clean up manually to prevent this. - repo.unfiltered().changelog.strip(oldtiprev, tr) - + _finishunshelve(repo, oldtiprev, tr) unshelvecleanup(ui, repo, basename, opts) - - _aborttransaction(repo) finally: ui.quiet = oldquiet if tr: @@ -912,9 +967,9 @@ elif checkopt('list'): return listcmd(ui, repo, pats, opts) elif checkopt('patch'): - return singlepatchcmds(ui, repo, pats, opts, subcommand='patch') + return patchcmds(ui, repo, pats, opts, subcommand='patch') elif checkopt('stat'): - return singlepatchcmds(ui, repo, pats, opts, subcommand='stat') + return patchcmds(ui, repo, pats, opts, subcommand='stat') else: return createcmd(ui, repo, pats, opts) diff -Nru mercurial-4.0.2~trusty1/hgext/win32mbcs.py mercurial-4.1~rc~trusty1/hgext/win32mbcs.py --- mercurial-4.0.2~trusty1/hgext/win32mbcs.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/hgext/win32mbcs.py 2017-01-18 16:54:51.000000000 +0000 @@ -53,6 +53,7 @@ from mercurial import ( encoding, error, + pycompat, ) # Note for extension authors: ONLY specify testedwith = 'ships-with-hg-core' for @@ -97,7 +98,7 @@ except UnicodeError: us = s if us and us[-1] not in ':/\\': - s += os.sep + s += pycompat.ossep return s @@ -138,10 +139,7 @@ func = getattr(module, name) def f(*args, **kwds): return wrapper(func, args, kwds) - try: - f.__name__ = func.__name__ # fails with Python 2.3 - except Exception: - pass + f.__name__ = func.__name__ setattr(module, name, f) # List of functions to be wrapped. @@ -171,7 +169,7 @@ def extsetup(ui): # TODO: decide use of config section for this extension if ((not os.path.supports_unicode_filenames) and - (sys.platform != 'cygwin')): + (pycompat.sysplatform != 'cygwin')): ui.warn(_("[win32mbcs] cannot activate on this platform.\n")) return # determine encoding for filename @@ -181,7 +179,7 @@ if _encoding.lower() in problematic_encodings.split(): for f in funcs.split(): wrapname(f, wrapper) - if os.name == 'nt': + if pycompat.osname == 'nt': for f in winfuncs.split(): wrapname(f, wrapper) wrapname("mercurial.osutil.listdir", wrapperforlistdir) diff -Nru mercurial-4.0.2~trusty1/.hgignore mercurial-4.1~rc~trusty1/.hgignore --- mercurial-4.0.2~trusty1/.hgignore 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/.hgignore 2017-01-18 16:54:51.000000000 +0000 @@ -49,6 +49,7 @@ tags cscope.* .idea/* +.asv/* i18n/hg.pot locale/*/LC_MESSAGES/hg.mo hgext/__index__.py diff -Nru mercurial-4.0.2~trusty1/.hgsigs mercurial-4.1~rc~trusty1/.hgsigs --- mercurial-4.0.2~trusty1/.hgsigs 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/.hgsigs 2017-01-18 16:54:51.000000000 +0000 @@ -136,3 +136,4 @@ 438173c415874f6ac653efc1099dec9c9150e90f 0 iQIVAwUAWAZ3okemf/qjRqrOAQj89xAAw/6QZ07yqvH+aZHeGQfgJ/X1Nze/hSMzkqbwGkuUOWD5ztN8+c39EXCn8JlqyLUPD7uGzhTV0299k5fGRihLIseXr0hy/cvVW16uqfeKJ/4/qL9zLS3rwSAgWbaHd1s6UQZVfGCb8V6oC1dkJxfrE9h6kugBqV97wStIRxmCpMDjsFv/zdNwsv6eEdxbiMilLn2/IbWXFOVKJzzv9iEY5Pu5McFR+nnrMyUZQhyGtVPLSkoEPsOysorfCZaVLJ6MnVaJunp9XEv94Pqx9+k+shsQvJHWkc0Nnb6uDHZYkLR5v2AbFsbJ9jDHsdr9A7qeQTiZay7PGI0uPoIrkmLya3cYbU1ADhwloAeQ/3gZLaJaKEjrXcFSsz7AZ9yq74rTwiPulF8uqZxJUodk2m/zy83HBrxxp/vgxWJ5JP2WXPtB8qKY+05umAt4rQS+fd2H/xOu2V2d5Mq1WmgknLBLC0ItaNaf91sSHtgEy22GtcvWQE7S6VWU1PoSYmOLITdJKAsmb7Eq+yKDW9nt0lOpUu2wUhBGctlgXgcWOmJP6gL6edIg66czAkVBp/fpKNl8Z/A0hhpuH7nW7GW/mzLVQnc+JW4wqUVkwlur3NRfvSt5ZyTY/SaR++nRf62h7PHIjU+f0kWQRdCcEQ0X38b8iAjeXcsOW8NCOPpm0zcz3i8= eab27446995210c334c3d06f1a659e3b9b5da769 0 iQIcBAABCAAGBQJYGNsXAAoJELnJ3IJKpb3Vf30QAK/dq5vEHEkufLGiYxxkvIyiRaswS+8jamXeHMQrdK8CuokcQYhEv9xiUI6FMIoX4Zc0xfoFCBc+X4qE+Ed9SFYWgQkDs/roJq1C1mTYA+KANMqJkDt00QZq536snFQvjCXAA5fwR/DpgGOOuGMRfvbjh7x8mPyVoPr4HDQCGFXnTYdn193HpTOqUsipzIV5OJqQ9p0sfJjwKP4ZfD0tqqdjTkNwMyJuwuRaReXFvGGCjH2PqkZE/FwQG0NJJjt0xaMUmv5U5tXHC9tEVobVV/qEslqfbH2v1YPF5d8Jmdn7F76FU5J0nTd+3rIVjYGYSt01cR6wtGnzvr/7kw9kbChw4wYhXxnmIALSd48FpA1qWjlPcAdHfUUwObxOxfqmlnBGtAQFK+p5VXCsxDZEIT9MSxscfCjyDQZpkY5S5B3PFIRg6V9bdl5a4rEt27aucuKTHj1Ok2vip4WfaIKk28YMjjzuOQRbr6Pp7mJcCC1/ERHUJdLsaQP+dy18z6XbDjX3O2JDRNYbCBexQyV/Kfrt5EOS5fXiByQUHv+PyR+9Ju6QWkkcFBfgsxq25kFl+eos4V9lxPOY5jDpw2BWu9TyHtTWkjL/YxDUGwUO9WA/WzrcT4skr9FYrFV/oEgi8MkwydC0cFICDfd6tr9upqkkr1W025Im1UBXXJ89bTVj b3b1ae98f6a0e14c1e1ba806a6c18e193b6dae5c 0 iQIVAwUAWECEaEemf/qjRqrOAQjuZw/+IWJKnKOsaUMcB9ly3Fo/eskqDL6A0j69IXTJDeBDGMoyGbQU/gZyX2yc6Sw3EhwTSCXu5vKpzg3a6e8MNrC1iHqli4wJ/jPY7XtmiqTYDixdsBLNk46VfOi73ooFe08wVDSNB65xpZsrtPDSioNmQ2kSJwSHb71UlauS4xGkM74vuDpWvX5OZRSfBqMh6NjG5RwBBnS8mzA0SW2dCI2jSc5SCGIzIZpzM0xUN21xzq0YQbrk9qEsmi7ks0eowdhUjeET2wSWwhOK4jS4IfMyRO7KueUB05yHs4mChj9kNFNWtSzXKwKBQbZzwO/1Y7IJjU+AsbWkiUu+6ipqBPQWzS28gCwGOrv5BcIJS+tzsvLUKWgcixyfy5UAqJ32gCdzKC54FUpT2zL6Ad0vXGM6WkpZA7yworN4RCFPexXbi0x2GSTLG8PyIoZ4Iwgtj5NtsEDHrz0380FxgnKUIC3ny2SVuPlyD+9wepD3QYcxdRk1BIzcFT9ZxNlgil3IXRVPwVejvQ/zr6/ILdhBnZ8ojjvVCy3b86B1OhZj/ZByYo5QaykVqWl0V9vJOZlZfvOpm2HiDhm/2uNrVWxG4O6EwhnekAdaJYmeLq1YbhIfGA6KVOaB9Yi5A5BxK9QGXBZ6sLj+dIUD3QR47r9yAqVQE8Gr/Oh6oQXBQqOQv7WzBBs= +e69874dc1f4e142746ff3df91e678a09c6fc208c 0 iQIVAwUAWG0oGUemf/qjRqrOAQh3uhAAu4TN7jkkgH7Hxn8S1cB6Ru0x8MQutzzzpjShhsE/G7nzCxsZ5eWdJ5ItwXmKhunb7T0og54CGcTxfmdPtCI7AhhHh9/TM2Hv1EBcsXCiwjG8E+P6X1UJkijgTGjNWuCvEDOsQAvgywslECBNnXp2QA5I5UdCMeqDdTAb8ujvbD8I4pxUx1xXKY18DgQGJh13mRlfkEVnPxUi2n8emnwPLjbVVkVISkMFUkaOl8a4fOeZC1xzDpoQocoH2Q8DYa9RCPPSHHSYPNMWGCdNGN2CoAurcHWWvc7jNU28/tBhTazfFv8LYh63lLQ8SIIPZHJAOxo45ufMspzUfNgoD6y3vlF5aW7DpdxwYHnueh7S1Fxgtd9cOnxmxQsgiF4LK0a+VXOi/Tli/fivZHDRCGHJvJgsMQm7pzkay9sGohes6jAnsOv2E8DwFC71FO/btrAp07IRFxH9WhUeMsXLMS9oBlubMxMM58M+xzSKApK6bz2MkLsx9cewmfmfbJnRIK1xDv+J+77pWWNGlxCCjl1WU+aA3M7G8HzwAqjL75ASOWtBrJlFXvlLgzobwwetg6cm44Rv1P39i3rDySZvi4BDlOQHWFupgMKiXnZ1PeL7eBDs/aawrE0V2ysNkf9An+XJZkos2JSLPWcoNigfXNUu5c1AqsERvHA246XJzqvCEK8= diff -Nru mercurial-4.0.2~trusty1/.hgtags mercurial-4.1~rc~trusty1/.hgtags --- mercurial-4.0.2~trusty1/.hgtags 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/.hgtags 2017-01-18 16:54:51.000000000 +0000 @@ -149,3 +149,4 @@ 438173c415874f6ac653efc1099dec9c9150e90f 4.0-rc eab27446995210c334c3d06f1a659e3b9b5da769 4.0 b3b1ae98f6a0e14c1e1ba806a6c18e193b6dae5c 4.0.1 +e69874dc1f4e142746ff3df91e678a09c6fc208c 4.0.2 diff -Nru mercurial-4.0.2~trusty1/i18n/ja.po mercurial-4.1~rc~trusty1/i18n/ja.po --- mercurial-4.0.2~trusty1/i18n/ja.po 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/i18n/ja.po 2017-01-18 16:54:51.000000000 +0000 @@ -6741,7 +6741,7 @@ msgstr "" msgid "" -"Scripts are run asychronously as detached daemon processes; mercurial will\n" +"Scripts are run asynchronously as detached daemon processes; mercurial will\n" "not ensure that they exit cleanly." msgstr "" @@ -10360,7 +10360,7 @@ msgstr "shelve 状態管理ファイルが破損しています" msgid "please run hg unshelve --abort to abort unshelve operation" -msgstr "'hg unshelve --abort' を実施して unshleve 操作を中断してください" +msgstr "'hg unshelve --abort' を実施して unshelve 操作を中断してください" msgid "" "could not read shelved state file, your working copy may be in an unexpected " @@ -14194,7 +14194,7 @@ msgstr "トランザクション実施途中は廃止マーカを破棄できません" #, python-format -msgid "deleted %i obsolescense markers\n" +msgid "deleted %i obsolescence markers\n" msgstr "%i 個の廃止マーカを破棄\n" #, python-format @@ -32661,8 +32661,8 @@ msgid "revset expects one or more arguments" msgstr "revset の引数は1個以上です" -msgid ":rstdoc(text, style): Format ReStructuredText." -msgstr ":rstdoc(text, style): 出力を ReStructuredText として整形します。" +msgid ":rstdoc(text, style): Format reStructuredText." +msgstr ":rstdoc(text, style): 出力を reStructuredText として整形します。" #. i18n: "rstdoc" is a keyword msgid "rstdoc expects two arguments" diff -Nru mercurial-4.0.2~trusty1/i18n/pt_BR.po mercurial-4.1~rc~trusty1/i18n/pt_BR.po --- mercurial-4.0.2~trusty1/i18n/pt_BR.po 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/i18n/pt_BR.po 2017-01-18 16:54:51.000000000 +0000 @@ -6803,7 +6803,7 @@ "falhasse." msgid "" -"Scripts are run asychronously as detached daemon processes; mercurial will\n" +"Scripts are run asynchronously as detached daemon processes; mercurial will\n" "not ensure that they exit cleanly." msgstr "" "Os scripts são executados assincronamente como processos desanexados;\n" @@ -14480,7 +14480,7 @@ "não é possível apagar marcações de obsolescência durante uma transação." #, python-format -msgid "deleted %i obsolescense markers\n" +msgid "deleted %i obsolescence markers\n" msgstr "%i marcações de obsolescência apagadas\n" #, python-format @@ -33571,8 +33571,8 @@ msgid "revset expects one or more arguments" msgstr "revset espera um ou mais argumentos" -msgid ":rstdoc(text, style): Format ReStructuredText." -msgstr ":rstdoc(texto, estilo): Formata ReStructuredText." +msgid ":rstdoc(text, style): Format reStructuredText." +msgstr ":rstdoc(texto, estilo): Formata reStructuredText." #. i18n: "rstdoc" is a keyword msgid "rstdoc expects two arguments" diff -Nru mercurial-4.0.2~trusty1/Makefile mercurial-4.1~rc~trusty1/Makefile --- mercurial-4.0.2~trusty1/Makefile 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/Makefile 2017-01-18 16:54:51.000000000 +0000 @@ -195,18 +195,18 @@ docker-ubuntu-trusty-ppa: contrib/docker/ubuntu-trusty contrib/dockerdeb ubuntu trusty --source-only -docker-ubuntu-wily: contrib/docker/ubuntu-wily - contrib/dockerdeb ubuntu wily - -docker-ubuntu-wily-ppa: contrib/docker/ubuntu-wily - contrib/dockerdeb ubuntu wily --source-only - docker-ubuntu-xenial: contrib/docker/ubuntu-xenial contrib/dockerdeb ubuntu xenial docker-ubuntu-xenial-ppa: contrib/docker/ubuntu-xenial contrib/dockerdeb ubuntu xenial --source-only +docker-ubuntu-yakkety: contrib/docker/ubuntu-yakkety + contrib/dockerdeb ubuntu yakkety + +docker-ubuntu-yakkety-ppa: contrib/docker/ubuntu-yakkety + contrib/dockerdeb ubuntu yakkety --source-only + fedora20: mkdir -p packages/fedora20 contrib/buildrpm diff -Nru mercurial-4.0.2~trusty1/mercurial/archival.py mercurial-4.1~rc~trusty1/mercurial/archival.py --- mercurial-4.0.2~trusty1/mercurial/archival.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/archival.py 2017-01-18 16:54:51.000000000 +0000 @@ -141,7 +141,7 @@ self.mtime = mtime self.fileobj = None - def taropen(name, mode, fileobj=None): + def taropen(mode, name='', fileobj=None): if kind == 'gz': mode = mode[0] if not fileobj: @@ -155,10 +155,9 @@ return tarfile.open(name, mode + kind, fileobj) if isinstance(dest, str): - self.z = taropen(dest, mode='w:') + self.z = taropen('w:', name=dest) else: - # Python 2.5-2.5.1 have a regression that requires a name arg - self.z = taropen(name='', mode='w|', fileobj=dest) + self.z = taropen('w|', fileobj=dest) def addfile(self, name, mode, islink, data): i = tarfile.TarInfo(name) diff -Nru mercurial-4.0.2~trusty1/mercurial/bdiff.c mercurial-4.1~rc~trusty1/mercurial/bdiff.c --- mercurial-4.0.2~trusty1/mercurial/bdiff.c 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/bdiff.c 2017-01-18 16:54:51.000000000 +0000 @@ -17,6 +17,10 @@ #include "bitmanipulation.h" #include "bdiff.h" +/* Hash implementation from diffutils */ +#define ROL(v, n) ((v) << (n) | (v) >> (sizeof(v) * CHAR_BIT - (n))) +#define HASH(h, c) ((c) + ROL(h ,7)) + struct pos { int pos, len; }; @@ -31,9 +35,11 @@ /* count the lines */ i = 1; /* extra line for sentinel */ - for (p = a; p < a + len; p++) - if (*p == '\n' || p == plast) + for (p = a; p < plast; p++) + if (*p == '\n') i++; + if (p == plast) + i++; *lr = l = (struct bdiff_line *)malloc(sizeof(struct bdiff_line) * i); if (!l) @@ -41,11 +47,10 @@ /* build the line array and calculate hashes */ hash = 0; - for (p = a; p < a + len; p++) { - /* Leonid Yuriev's hash */ - hash = (hash * 1664525) + (unsigned char)*p + 1013904223; + for (p = a; p < plast; p++) { + hash = HASH(hash, *p); - if (*p == '\n' || p == plast) { + if (*p == '\n') { l->hash = hash; hash = 0; l->len = p - b + 1; @@ -56,6 +61,15 @@ } } + if (p == plast) { + hash = HASH(hash, *p); + l->hash = hash; + l->len = p - b + 1; + l->l = b; + l->n = INT_MAX; + l++; + } + /* set up a sentinel */ l->hash = 0; l->len = 0; @@ -138,7 +152,7 @@ struct pos *pos, int a1, int a2, int b1, int b2, int *omi, int *omj) { - int mi = a1, mj = b1, mk = 0, i, j, k, half; + int mi = a1, mj = b1, mk = 0, i, j, k, half, bhalf; /* window our search on large regions to better bound worst-case performance. by choosing a window at the end, we @@ -146,7 +160,8 @@ if (a2 - a1 > 30000) a1 = a2 - 30000; - half = (a1 + a2) / 2; + half = (a1 + a2 - 1) / 2; + bhalf = (b1 + b2 - 1) / 2; for (i = a1; i < a2; i++) { /* skip all lines in b after the current block */ @@ -172,10 +187,20 @@ /* best match so far? we prefer matches closer to the middle to balance recursion */ - if (k > mk || (k == mk && (i <= mi || i < half))) { + if (k > mk) { + /* a longer match */ mi = i; mj = j; mk = k; + } else if (k == mk) { + if (i > mi && i <= half && j > b1) { + /* same match but closer to half */ + mi = i; + mj = j; + } else if (i == mi && (mj > bhalf || i == a1)) { + /* same i but best earlier j */ + mj = j; + } } } } diff -Nru mercurial-4.0.2~trusty1/mercurial/bdiff_module.c mercurial-4.1~rc~trusty1/mercurial/bdiff_module.c --- mercurial-4.0.2~trusty1/mercurial/bdiff_module.c 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/bdiff_module.c 2017-01-18 16:54:51.000000000 +0000 @@ -61,12 +61,12 @@ static PyObject *bdiff(PyObject *self, PyObject *args) { - char *sa, *sb, *rb; + char *sa, *sb, *rb, *ia, *ib; PyObject *result = NULL; struct bdiff_line *al, *bl; struct bdiff_hunk l, *h; int an, bn, count; - Py_ssize_t len = 0, la, lb; + Py_ssize_t len = 0, la, lb, li = 0, lcommon = 0, lmax; PyThreadState *_save; l.next = NULL; @@ -80,8 +80,17 @@ } _save = PyEval_SaveThread(); - an = bdiff_splitlines(sa, la, &al); - bn = bdiff_splitlines(sb, lb, &bl); + + lmax = la > lb ? lb : la; + for (ia = sa, ib = sb; + li < lmax && *ia == *ib; + ++li, ++ia, ++ib) + if (*ia == '\n') + lcommon = li + 1; + /* we can almost add: if (li == lmax) lcommon = li; */ + + an = bdiff_splitlines(sa + lcommon, la - lcommon, &al); + bn = bdiff_splitlines(sb + lcommon, lb - lcommon, &bl); if (!al || !bl) goto nomem; @@ -112,8 +121,8 @@ for (h = l.next; h; h = h->next) { if (h->a1 != la || h->b1 != lb) { len = bl[h->b1].l - bl[lb].l; - putbe32((uint32_t)(al[la].l - al->l), rb); - putbe32((uint32_t)(al[h->a1].l - al->l), rb + 4); + putbe32((uint32_t)(al[la].l + lcommon - al->l), rb); + putbe32((uint32_t)(al[h->a1].l + lcommon - al->l), rb + 4); putbe32((uint32_t)len, rb + 8); memcpy(rb + 12, bl[lb].l, len); rb += 12 + len; diff -Nru mercurial-4.0.2~trusty1/mercurial/bookmarks.py mercurial-4.1~rc~trusty1/mercurial/bookmarks.py --- mercurial-4.0.2~trusty1/mercurial/bookmarks.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/bookmarks.py 2017-01-18 16:54:51.000000000 +0000 @@ -8,7 +8,6 @@ from __future__ import absolute_import import errno -import os from .i18n import _ from .node import ( @@ -31,7 +30,7 @@ may need to tweak this behavior further. """ bkfile = None - if 'HG_PENDING' in os.environ: + if 'HG_PENDING' in encoding.environ: try: bkfile = repo.vfs('bookmarks.pending') except IOError as inst: @@ -284,17 +283,21 @@ lockmod.release(tr, lock) return update -def listbookmarks(repo): +def listbinbookmarks(repo): # We may try to list bookmarks on a repo type that does not # support it (e.g., statichttprepository). marks = getattr(repo, '_bookmarks', {}) - d = {} hasnode = repo.changelog.hasnode for k, v in marks.iteritems(): # don't expose local divergent bookmarks if hasnode(v) and ('@' not in k or k.endswith('@')): - d[k] = hex(v) + yield k, v + +def listbookmarks(repo): + d = {} + for book, node in listbinbookmarks(repo): + d[book] = hex(node) return d def pushbookmark(repo, key, old, new): @@ -319,8 +322,7 @@ finally: lockmod.release(tr, l, w) -def compare(repo, srcmarks, dstmarks, - srchex=None, dsthex=None, targets=None): +def comparebookmarks(repo, srcmarks, dstmarks, targets=None): '''Compare bookmarks between srcmarks and dstmarks This returns tuple "(addsrc, adddst, advsrc, advdst, diverge, @@ -343,19 +345,9 @@ Changeset IDs of tuples in "addsrc", "adddst", "differ" or "invalid" list may be unknown for repo. - This function expects that "srcmarks" and "dstmarks" return - changeset ID in 40 hexadecimal digit string for specified - bookmark. If not so (e.g. bmstore "repo._bookmarks" returning - binary value), "srchex" or "dsthex" should be specified to convert - into such form. - If "targets" is specified, only bookmarks listed in it are examined. ''' - if not srchex: - srchex = lambda x: x - if not dsthex: - dsthex = lambda x: x if targets: bset = set(targets) @@ -377,14 +369,14 @@ for b in sorted(bset): if b not in srcmarks: if b in dstmarks: - adddst((b, None, dsthex(dstmarks[b]))) + adddst((b, None, dstmarks[b])) else: invalid((b, None, None)) elif b not in dstmarks: - addsrc((b, srchex(srcmarks[b]), None)) + addsrc((b, srcmarks[b], None)) else: - scid = srchex(srcmarks[b]) - dcid = dsthex(dstmarks[b]) + scid = srcmarks[b] + dcid = dstmarks[b] if scid == dcid: same((b, scid, dcid)) elif scid in repo and dcid in repo: @@ -435,11 +427,17 @@ return None +def unhexlifybookmarks(marks): + binremotemarks = {} + for name, node in marks.items(): + binremotemarks[name] = bin(node) + return binremotemarks + def updatefromremote(ui, repo, remotemarks, path, trfunc, explicit=()): ui.debug("checking for updated bookmarks\n") localmarks = repo._bookmarks (addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same - ) = compare(repo, remotemarks, localmarks, dsthex=hex) + ) = comparebookmarks(repo, remotemarks, localmarks) status = ui.status warn = ui.warn @@ -450,15 +448,15 @@ changed = [] for b, scid, dcid in addsrc: if scid in repo: # add remote bookmarks for changes we already have - changed.append((b, bin(scid), status, + changed.append((b, scid, status, _("adding remote bookmark %s\n") % (b))) elif b in explicit: explicit.remove(b) ui.warn(_("remote bookmark %s points to locally missing %s\n") - % (b, scid[:12])) + % (b, hex(scid)[:12])) for b, scid, dcid in advsrc: - changed.append((b, bin(scid), status, + changed.append((b, scid, status, _("updating bookmark %s\n") % (b))) # remove normal movement from explicit set explicit.difference_update(d[0] for d in changed) @@ -466,13 +464,12 @@ for b, scid, dcid in diverge: if b in explicit: explicit.discard(b) - changed.append((b, bin(scid), status, + changed.append((b, scid, status, _("importing bookmark %s\n") % (b))) else: - snode = bin(scid) - db = _diverge(ui, b, path, localmarks, snode) + db = _diverge(ui, b, path, localmarks, scid) if db: - changed.append((db, snode, warn, + changed.append((db, scid, warn, _("divergent bookmark %s stored as %s\n") % (b, db))) else: @@ -481,13 +478,13 @@ for b, scid, dcid in adddst + advdst: if b in explicit: explicit.discard(b) - changed.append((b, bin(scid), status, + changed.append((b, scid, status, _("importing bookmark %s\n") % (b))) for b, scid, dcid in differ: if b in explicit: explicit.remove(b) ui.warn(_("remote bookmark %s points to locally missing %s\n") - % (b, scid[:12])) + % (b, hex(scid)[:12])) if changed: tr = trfunc() @@ -501,8 +498,8 @@ ''' ui.status(_("searching for changed bookmarks\n")) - r = compare(repo, other.listkeys('bookmarks'), repo._bookmarks, - dsthex=hex) + remotemarks = unhexlifybookmarks(other.listkeys('bookmarks')) + r = comparebookmarks(repo, remotemarks, repo._bookmarks) addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = r incomings = [] @@ -518,16 +515,16 @@ incomings.append(" %-25s %s\n" % (b, getid(id))) for b, scid, dcid in addsrc: # i18n: "added" refers to a bookmark - add(b, scid, _('added')) + add(b, hex(scid), _('added')) for b, scid, dcid in advsrc: # i18n: "advanced" refers to a bookmark - add(b, scid, _('advanced')) + add(b, hex(scid), _('advanced')) for b, scid, dcid in diverge: # i18n: "diverged" refers to a bookmark - add(b, scid, _('diverged')) + add(b, hex(scid), _('diverged')) for b, scid, dcid in differ: # i18n: "changed" refers to a bookmark - add(b, scid, _('changed')) + add(b, hex(scid), _('changed')) if not incomings: ui.status(_("no changed bookmarks found\n")) @@ -543,8 +540,8 @@ ''' ui.status(_("searching for changed bookmarks\n")) - r = compare(repo, repo._bookmarks, other.listkeys('bookmarks'), - srchex=hex) + remotemarks = unhexlifybookmarks(other.listkeys('bookmarks')) + r = comparebookmarks(repo, repo._bookmarks, remotemarks) addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = r outgoings = [] @@ -560,19 +557,19 @@ outgoings.append(" %-25s %s\n" % (b, getid(id))) for b, scid, dcid in addsrc: # i18n: "added refers to a bookmark - add(b, scid, _('added')) + add(b, hex(scid), _('added')) for b, scid, dcid in adddst: # i18n: "deleted" refers to a bookmark add(b, ' ' * 40, _('deleted')) for b, scid, dcid in advsrc: # i18n: "advanced" refers to a bookmark - add(b, scid, _('advanced')) + add(b, hex(scid), _('advanced')) for b, scid, dcid in diverge: # i18n: "diverged" refers to a bookmark - add(b, scid, _('diverged')) + add(b, hex(scid), _('diverged')) for b, scid, dcid in differ: # i18n: "changed" refers to a bookmark - add(b, scid, _('changed')) + add(b, hex(scid), _('changed')) if not outgoings: ui.status(_("no changed bookmarks found\n")) @@ -588,8 +585,8 @@ This returns "(# of incoming, # of outgoing)" tuple. ''' - r = compare(repo, other.listkeys('bookmarks'), repo._bookmarks, - dsthex=hex) + remotemarks = unhexlifybookmarks(other.listkeys('bookmarks')) + r = comparebookmarks(repo, remotemarks, repo._bookmarks) addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = r return (len(addsrc), len(adddst)) diff -Nru mercurial-4.0.2~trusty1/mercurial/bundle2.py mercurial-4.1~rc~trusty1/mercurial/bundle2.py --- mercurial-4.0.2~trusty1/mercurial/bundle2.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/bundle2.py 2017-01-18 16:54:51.000000000 +0000 @@ -485,11 +485,11 @@ return '\n'.join(chunks) bundletypes = { - "": ("", None), # only when using unbundle on ssh and old http servers + "": ("", 'UN'), # only when using unbundle on ssh and old http servers # since the unification ssh accepts a header but there # is no capability signaling it. "HG20": (), # special-cased below - "HG10UN": ("HG10UN", None), + "HG10UN": ("HG10UN", 'UN'), "HG10BZ": ("HG10", 'BZ'), "HG10GZ": ("HG10GZ", 'GZ'), } @@ -511,15 +511,17 @@ self._params = [] self._parts = [] self.capabilities = dict(capabilities) - self._compressor = util.compressors[None]() + self._compengine = util.compengines.forbundletype('UN') + self._compopts = None - def setcompression(self, alg): + def setcompression(self, alg, compopts=None): """setup core part compression to """ - if alg is None: + if alg in (None, 'UN'): return assert not any(n.lower() == 'Compression' for n, v in self._params) self.addparam('Compression', alg) - self._compressor = util.compressors[alg]() + self._compengine = util.compengines.forbundletype(alg) + self._compopts = compopts @property def nbparts(self): @@ -571,12 +573,9 @@ yield _pack(_fstreamparamsize, len(param)) if param: yield param - # starting compression - for chunk in self._getcorechunk(): - data = self._compressor.compress(chunk) - if data: - yield data - yield self._compressor.flush() + for chunk in self._compengine.compressstream(self._getcorechunk(), + self._compopts): + yield chunk def _paramchunk(self): """return a encoded version of all stream parameters""" @@ -680,7 +679,7 @@ def __init__(self, ui, fp): """If header is specified, we do not read it out of the stream.""" self.ui = ui - self._decompressor = util.decompressors[None] + self._compengine = util.compengines.forbundletype('UN') self._compressed = None super(unbundle20, self).__init__(fp) @@ -754,9 +753,9 @@ params = self._readexact(paramssize) self._processallparams(params) yield params - assert self._decompressor is util.decompressors[None] + assert self._compengine.bundletype == 'UN' # From there, payload might need to be decompressed - self._fp = self._decompressor(self._fp) + self._fp = self._compengine.decompressorreader(self._fp) emptycount = 0 while emptycount < 2: # so we can brainlessly loop @@ -780,7 +779,7 @@ # make sure param have been loaded self.params # From there, payload need to be decompressed - self._fp = self._decompressor(self._fp) + self._fp = self._compengine.decompressorreader(self._fp) indebug(self.ui, 'start extraction of bundle2 parts') headerblock = self._readpartheader() while headerblock is not None: @@ -822,10 +821,10 @@ @b2streamparamhandler('compression') def processcompression(unbundler, param, value): """read compression parameter and install payload decompression""" - if value not in util.decompressors: + if value not in util.compengines.supportedbundletypes: raise error.BundleUnknownFeatureError(params=(param,), values=(value,)) - unbundler._decompressor = util.decompressors[value] + unbundler._compengine = util.compengines.forbundletype(value) if value is not None: unbundler._compressed = True @@ -1293,7 +1292,8 @@ obscaps = caps.get('obsmarkers', ()) return [int(c[1:]) for c in obscaps if c.startswith('V')] -def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None): +def writebundle(ui, cg, filename, bundletype, vfs=None, compression=None, + compopts=None): """Write a bundle file and return its filename. Existing files will not be overwritten. @@ -1304,7 +1304,7 @@ if bundletype == "HG20": bundle = bundle20(ui) - bundle.setcompression(compression) + bundle.setcompression(compression, compopts) part = bundle.newpart('changegroup', data=cg.getchunks()) part.addparam('version', cg.version) if 'clcount' in cg.extras: @@ -1318,18 +1318,14 @@ raise error.Abort(_('old bundle types only supports v1 ' 'changegroups')) header, comp = bundletypes[bundletype] - if comp not in util.compressors: + if comp not in util.compengines.supportedbundletypes: raise error.Abort(_('unknown stream compression type: %s') % comp) - z = util.compressors[comp]() - subchunkiter = cg.getchunks() + compengine = util.compengines.forbundletype(comp) def chunkiter(): yield header - for chunk in subchunkiter: - data = z.compress(chunk) - if data: - yield data - yield z.flush() + for chunk in compengine.compressstream(cg.getchunks(), compopts): + yield chunk chunkiter = chunkiter() # parse the changegroup data, otherwise we will block diff -Nru mercurial-4.0.2~trusty1/mercurial/bundlerepo.py mercurial-4.1~rc~trusty1/mercurial/bundlerepo.py --- mercurial-4.0.2~trusty1/mercurial/bundlerepo.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/bundlerepo.py 2017-01-18 16:54:51.000000000 +0000 @@ -35,6 +35,7 @@ node as nodemod, pathutil, phases, + pycompat, revlog, scmutil, util, @@ -116,7 +117,7 @@ return mdiff.textdiff(self.revision(self.node(rev1)), self.revision(self.node(rev2))) - def revision(self, nodeorrev): + def revision(self, nodeorrev, raw=False): """return an uncompressed revision of a given node or revision number. """ @@ -147,7 +148,10 @@ delta = self._chunk(chain.pop()) text = mdiff.patches(text, [delta]) - self._checkhash(text, node, rev) + text, validatehash = self._processflags(text, self.flags(rev), + 'read', raw=raw) + if validatehash: + self.checkhash(text, node, rev=rev) self._cache = (node, rev, text) return text @@ -187,9 +191,9 @@ finally: self.filteredrevs = oldfilter -class bundlemanifest(bundlerevlog, manifest.manifest): +class bundlemanifest(bundlerevlog, manifest.manifestrevlog): def __init__(self, opener, bundle, linkmapper, dirlogstarts=None, dir=''): - manifest.manifest.__init__(self, opener, dir=dir) + manifest.manifestrevlog.__init__(self, opener, dir=dir) bundlerevlog.__init__(self, opener, self.indexfile, bundle, linkmapper) if dirlogstarts is None: @@ -207,7 +211,7 @@ if node in self.fulltextcache: result = self.fulltextcache[node].tostring() else: - result = manifest.manifest.revision(self, nodeorrev) + result = manifest.manifestrevlog.revision(self, nodeorrev) return result def dirlog(self, d): @@ -359,7 +363,7 @@ @localrepo.unfilteredpropertycache def filestart(self): - self.manifest + self.manifestlog return self.filestart def url(self): @@ -392,7 +396,7 @@ return bundlepeer(self) def getcwd(self): - return os.getcwd() # always outside the repo + return pycompat.getcwd() # always outside the repo # Check if parents exist in localrepo before setting def setparents(self, p1, p2=nullid): @@ -412,13 +416,13 @@ parentpath = ui.config("bundle", "mainreporoot", "") if not parentpath: # try to find the correct path to the working directory repo - parentpath = cmdutil.findrepo(os.getcwd()) + parentpath = cmdutil.findrepo(pycompat.getcwd()) if parentpath is None: parentpath = '' if parentpath: # Try to make the full path relative so we get a nice, short URL. # In particular, we don't want temp dir names in test outputs. - cwd = os.getcwd() + cwd = pycompat.getcwd() if parentpath == cwd: parentpath = '' else: diff -Nru mercurial-4.0.2~trusty1/mercurial/cffi/bdiff.py mercurial-4.1~rc~trusty1/mercurial/cffi/bdiff.py --- mercurial-4.0.2~trusty1/mercurial/cffi/bdiff.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/cffi/bdiff.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,31 @@ +from __future__ import absolute_import + +import cffi +import os + +ffi = cffi.FFI() +ffi.set_source("_bdiff_cffi", + open(os.path.join(os.path.join(os.path.dirname(__file__), '..'), + 'bdiff.c')).read(), include_dirs=['mercurial']) +ffi.cdef(""" +struct bdiff_line { + int hash, n, e; + ssize_t len; + const char *l; +}; + +struct bdiff_hunk; +struct bdiff_hunk { + int a1, a2, b1, b2; + struct bdiff_hunk *next; +}; + +int bdiff_splitlines(const char *a, ssize_t len, struct bdiff_line **lr); +int bdiff_diff(struct bdiff_line *a, int an, struct bdiff_line *b, int bn, + struct bdiff_hunk *base); +void bdiff_freehunks(struct bdiff_hunk *l); +void free(void*); +""") + +if __name__ == '__main__': + ffi.compile() diff -Nru mercurial-4.0.2~trusty1/mercurial/cffi/mpatch.py mercurial-4.1~rc~trusty1/mercurial/cffi/mpatch.py --- mercurial-4.0.2~trusty1/mercurial/cffi/mpatch.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/cffi/mpatch.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,35 @@ +from __future__ import absolute_import + +import cffi +import os + +ffi = cffi.FFI() +mpatch_c = os.path.join(os.path.join(os.path.dirname(__file__), '..', + 'mpatch.c')) +ffi.set_source("_mpatch_cffi", open(mpatch_c).read(), + include_dirs=["mercurial"]) +ffi.cdef(""" + +struct mpatch_frag { + int start, end, len; + const char *data; +}; + +struct mpatch_flist { + struct mpatch_frag *base, *head, *tail; +}; + +extern "Python" struct mpatch_flist* cffi_get_next_item(void*, ssize_t); + +int mpatch_decode(const char *bin, ssize_t len, struct mpatch_flist** res); +ssize_t mpatch_calcsize(size_t len, struct mpatch_flist *l); +void mpatch_lfree(struct mpatch_flist *a); +static int mpatch_apply(char *buf, const char *orig, size_t len, + struct mpatch_flist *l); +struct mpatch_flist *mpatch_fold(void *bins, + struct mpatch_flist* (*get_next_item)(void*, ssize_t), + ssize_t start, ssize_t end); +""") + +if __name__ == '__main__': + ffi.compile() diff -Nru mercurial-4.0.2~trusty1/mercurial/cffi/osutil.py mercurial-4.1~rc~trusty1/mercurial/cffi/osutil.py --- mercurial-4.0.2~trusty1/mercurial/cffi/osutil.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/cffi/osutil.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,102 @@ +from __future__ import absolute_import + +import cffi + +ffi = cffi.FFI() +ffi.set_source("_osutil_cffi", """ +#include +#include +#include +#include +#include + +typedef struct val_attrs { + uint32_t length; + attribute_set_t returned; + attrreference_t name_info; + fsobj_type_t obj_type; + struct timespec mtime; + uint32_t accessmask; + off_t datalength; +} __attribute__((aligned(4), packed)) val_attrs_t; +""", include_dirs=['mercurial']) +ffi.cdef(''' + +typedef uint32_t attrgroup_t; + +typedef struct attrlist { + uint16_t bitmapcount; /* number of attr. bit sets in list */ + uint16_t reserved; /* (to maintain 4-byte alignment) */ + attrgroup_t commonattr; /* common attribute group */ + attrgroup_t volattr; /* volume attribute group */ + attrgroup_t dirattr; /* directory attribute group */ + attrgroup_t fileattr; /* file attribute group */ + attrgroup_t forkattr; /* fork attribute group */ + ...; +}; + +typedef struct attribute_set { + ...; +} attribute_set_t; + +typedef struct attrreference { + int attr_dataoffset; + int attr_length; + ...; +} attrreference_t; + +typedef int ... off_t; + +typedef struct val_attrs { + uint32_t length; + attribute_set_t returned; + attrreference_t name_info; + uint32_t obj_type; + struct timespec mtime; + uint32_t accessmask; + off_t datalength; + ...; +} val_attrs_t; + +/* the exact layout of the above struct will be figured out during build time */ + +typedef int ... time_t; + +typedef struct timespec { + time_t tv_sec; + ...; +}; + +int getattrlist(const char* path, struct attrlist * attrList, void * attrBuf, + size_t attrBufSize, unsigned int options); + +int getattrlistbulk(int dirfd, struct attrlist * attrList, void * attrBuf, + size_t attrBufSize, uint64_t options); + +#define ATTR_BIT_MAP_COUNT ... +#define ATTR_CMN_NAME ... +#define ATTR_CMN_OBJTYPE ... +#define ATTR_CMN_MODTIME ... +#define ATTR_CMN_ACCESSMASK ... +#define ATTR_CMN_ERROR ... +#define ATTR_CMN_RETURNED_ATTRS ... +#define ATTR_FILE_DATALENGTH ... + +#define VREG ... +#define VDIR ... +#define VLNK ... +#define VBLK ... +#define VCHR ... +#define VFIFO ... +#define VSOCK ... + +#define S_IFMT ... + +int open(const char *path, int oflag, int perm); +int close(int); + +#define O_RDONLY ... +''') + +if __name__ == '__main__': + ffi.compile() diff -Nru mercurial-4.0.2~trusty1/mercurial/changegroup.py mercurial-4.1~rc~trusty1/mercurial/changegroup.py --- mercurial-4.0.2~trusty1/mercurial/changegroup.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/changegroup.py 2017-01-18 16:54:51.000000000 +0000 @@ -137,14 +137,16 @@ _grouplistcount = 1 # One list of files after the manifests def __init__(self, fh, alg, extras=None): - if alg == 'UN': - alg = None # get more modern without breaking too much - if not alg in util.decompressors: + if alg is None: + alg = 'UN' + if alg not in util.compengines.supportedbundletypes: raise error.Abort(_('unknown stream compression type: %s') % alg) if alg == 'BZ': alg = '_truncatedBZ' - self._stream = util.decompressors[alg](fh) + + compengine = util.compengines.forbundletype(alg) + self._stream = compengine.decompressorreader(fh) self._type = alg self.extras = extras or {} self.callback = None @@ -152,7 +154,7 @@ # These methods (compressed, read, seek, tell) all appear to only # be used by bundlerepo, but it's a little hard to tell. def compressed(self): - return self._type is not None + return self._type is not None and self._type != 'UN' def read(self, l): return self._stream.read(l) def seek(self, pos): @@ -250,7 +252,7 @@ # no new manifest will be created and the manifest group will # be empty during the pull self.manifestheader() - repo.manifest.addgroup(self, revmap, trp) + repo.manifestlog._revlog.addgroup(self, revmap, trp) repo.ui.progress(_('manifests'), None) self.callback = None @@ -330,11 +332,12 @@ needfiles = {} if repo.ui.configbool('server', 'validate', default=False): + cl = repo.changelog + ml = repo.manifestlog # validate incoming csets have their manifests for cset in xrange(clstart, clend): - mfnode = repo.changelog.read( - repo.changelog.node(cset))[0] - mfest = repo.manifestlog[mfnode].readdelta() + mfnode = cl.changelogrevision(cset).manifest + mfest = ml[mfnode].readdelta() # store file nodes we must see for f, n in mfest.iteritems(): needfiles.setdefault(f, set()).add(n) @@ -479,7 +482,7 @@ # If we get here, there are directory manifests in the changegroup d = chunkdata["filename"] repo.ui.debug("adding %s revisions\n" % d) - dirlog = repo.manifest.dirlog(d) + dirlog = repo.manifestlog._revlog.dirlog(d) if not dirlog.addgroup(self, revmap, trp): raise error.Abort(_("received dir revlog group is empty")) @@ -587,7 +590,7 @@ def _packmanifests(self, dir, mfnodes, lookuplinknode): """Pack flat manifests into a changegroup stream.""" assert not dir - for chunk in self.group(mfnodes, self._repo.manifest, + for chunk in self.group(mfnodes, self._repo.manifestlog._revlog, lookuplinknode, units=_('manifests')): yield chunk @@ -676,7 +679,8 @@ def generatemanifests(self, commonrevs, clrevorder, fastpathlinkrev, mfs, fnodes): repo = self._repo - dirlog = repo.manifest.dirlog + mfl = repo.manifestlog + dirlog = mfl._revlog.dirlog tmfnodes = {'': mfs} # Callback for the manifest, used to collect linkrevs for filelog @@ -704,7 +708,7 @@ treemanifests to send. """ clnode = tmfnodes[dir][x] - mdata = dirlog(dir).readshallowfast(x) + mdata = mfl.get(dir, x).readfast(shallow=True) for p, n, fl in mdata.iterentries(): if fl == 't': # subdirectory manifest subdir = dir + p + '/' @@ -779,7 +783,7 @@ prefix = '' if revlog.iscensored(base) or revlog.iscensored(rev): try: - delta = revlog.revision(node) + delta = revlog.revision(node, raw=True) except error.CensoredNodeError as e: delta = e.tombstone if base == nullrev: @@ -788,7 +792,7 @@ baselen = revlog.rawsize(base) prefix = mdiff.replacediffheader(baselen, len(delta)) elif base == nullrev: - delta = revlog.revision(node) + delta = revlog.revision(node, raw=True) prefix = mdiff.trivialdiffheader(len(delta)) else: delta = revlog.revdiff(base, rev) @@ -850,8 +854,10 @@ def _packmanifests(self, dir, mfnodes, lookuplinknode): if dir: yield self.fileheader(dir) - for chunk in self.group(mfnodes, self._repo.manifest.dirlog(dir), - lookuplinknode, units=_('manifests')): + + dirlog = self._repo.manifestlog._revlog.dirlog(dir) + for chunk in self.group(mfnodes, dirlog, lookuplinknode, + units=_('manifests')): yield chunk def _manifestsdone(self): @@ -868,24 +874,21 @@ '03': (cg3packer, cg3unpacker), } -def allsupportedversions(ui): +def allsupportedversions(repo): versions = set(_packermap.keys()) - versions.discard('03') - if (ui.configbool('experimental', 'changegroup3') or - ui.configbool('experimental', 'treemanifest')): - versions.add('03') + if not (repo.ui.configbool('experimental', 'changegroup3') or + repo.ui.configbool('experimental', 'treemanifest') or + 'treemanifest' in repo.requirements): + versions.discard('03') return versions # Changegroup versions that can be applied to the repo def supportedincomingversions(repo): - versions = allsupportedversions(repo.ui) - if 'treemanifest' in repo.requirements: - versions.add('03') - return versions + return allsupportedversions(repo) # Changegroup versions that can be created from the repo def supportedoutgoingversions(repo): - versions = allsupportedversions(repo.ui) + versions = allsupportedversions(repo) if 'treemanifest' in repo.requirements: # Versions 01 and 02 support only flat manifests and it's just too # expensive to convert between the flat manifest and tree manifest on @@ -894,7 +897,6 @@ # support versions 01 and 02. versions.discard('01') versions.discard('02') - versions.add('03') return versions def safeversion(repo): diff -Nru mercurial-4.0.2~trusty1/mercurial/changelog.py mercurial-4.1~rc~trusty1/mercurial/changelog.py --- mercurial-4.0.2~trusty1/mercurial/changelog.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/changelog.py 2017-01-18 16:54:51.000000000 +0000 @@ -79,9 +79,10 @@ self.fp = fp self.offset = fp.tell() self.size = vfs.fstat(fp).st_size + self._end = self.size def end(self): - return self.size + len("".join(self.data)) + return self._end def tell(self): return self.offset def flush(self): @@ -121,6 +122,7 @@ def write(self, s): self.data.append(str(s)) self.offset += len(s) + self._end += len(s) def _divertopener(opener, target): """build an opener that writes in 'target.a' instead of 'target'""" diff -Nru mercurial-4.0.2~trusty1/mercurial/chgserver.py mercurial-4.1~rc~trusty1/mercurial/chgserver.py --- mercurial-4.0.2~trusty1/mercurial/chgserver.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/chgserver.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,588 @@ +# chgserver.py - command server extension for cHg +# +# Copyright 2011 Yuya Nishihara +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +"""command server extension for cHg + +'S' channel (read/write) + propagate ui.system() request to client + +'attachio' command + attach client's stdio passed by sendmsg() + +'chdir' command + change current directory + +'setenv' command + replace os.environ completely + +'setumask' command + set umask + +'validate' command + reload the config and check if the server is up to date + +Config +------ + +:: + + [chgserver] + idletimeout = 3600 # seconds, after which an idle server will exit + skiphash = False # whether to skip config or env change checks +""" + +from __future__ import absolute_import + +import errno +import hashlib +import inspect +import os +import re +import struct +import time + +from .i18n import _ + +from . import ( + commandserver, + encoding, + error, + extensions, + osutil, + pycompat, + util, +) + +_log = commandserver.log + +def _hashlist(items): + """return sha1 hexdigest for a list""" + return hashlib.sha1(str(items)).hexdigest() + +# sensitive config sections affecting confighash +_configsections = [ + 'alias', # affects global state commands.table + 'extdiff', # uisetup will register new commands + 'extensions', +] + +# sensitive environment variables affecting confighash +_envre = re.compile(r'''\A(?: + CHGHG + |HG(?:[A-Z].*)? + |LANG(?:UAGE)? + |LC_.* + |LD_.* + |PATH + |PYTHON.* + |TERM(?:INFO)? + |TZ + )\Z''', re.X) + +def _confighash(ui): + """return a quick hash for detecting config/env changes + + confighash is the hash of sensitive config items and environment variables. + + for chgserver, it is designed that once confighash changes, the server is + not qualified to serve its client and should redirect the client to a new + server. different from mtimehash, confighash change will not mark the + server outdated and exit since the user can have different configs at the + same time. + """ + sectionitems = [] + for section in _configsections: + sectionitems.append(ui.configitems(section)) + sectionhash = _hashlist(sectionitems) + envitems = [(k, v) for k, v in encoding.environ.iteritems() + if _envre.match(k)] + envhash = _hashlist(sorted(envitems)) + return sectionhash[:6] + envhash[:6] + +def _getmtimepaths(ui): + """get a list of paths that should be checked to detect change + + The list will include: + - extensions (will not cover all files for complex extensions) + - mercurial/__version__.py + - python binary + """ + modules = [m for n, m in extensions.extensions(ui)] + try: + from . import __version__ + modules.append(__version__) + except ImportError: + pass + files = [pycompat.sysexecutable] + for m in modules: + try: + files.append(inspect.getabsfile(m)) + except TypeError: + pass + return sorted(set(files)) + +def _mtimehash(paths): + """return a quick hash for detecting file changes + + mtimehash calls stat on given paths and calculate a hash based on size and + mtime of each file. mtimehash does not read file content because reading is + expensive. therefore it's not 100% reliable for detecting content changes. + it's possible to return different hashes for same file contents. + it's also possible to return a same hash for different file contents for + some carefully crafted situation. + + for chgserver, it is designed that once mtimehash changes, the server is + considered outdated immediately and should no longer provide service. + + mtimehash is not included in confighash because we only know the paths of + extensions after importing them (there is imp.find_module but that faces + race conditions). We need to calculate confighash without importing. + """ + def trystat(path): + try: + st = os.stat(path) + return (st.st_mtime, st.st_size) + except OSError: + # could be ENOENT, EPERM etc. not fatal in any case + pass + return _hashlist(map(trystat, paths))[:12] + +class hashstate(object): + """a structure storing confighash, mtimehash, paths used for mtimehash""" + def __init__(self, confighash, mtimehash, mtimepaths): + self.confighash = confighash + self.mtimehash = mtimehash + self.mtimepaths = mtimepaths + + @staticmethod + def fromui(ui, mtimepaths=None): + if mtimepaths is None: + mtimepaths = _getmtimepaths(ui) + confighash = _confighash(ui) + mtimehash = _mtimehash(mtimepaths) + _log('confighash = %s mtimehash = %s\n' % (confighash, mtimehash)) + return hashstate(confighash, mtimehash, mtimepaths) + +def _newchgui(srcui, csystem, attachio): + class chgui(srcui.__class__): + def __init__(self, src=None): + super(chgui, self).__init__(src) + if src: + self._csystem = getattr(src, '_csystem', csystem) + else: + self._csystem = csystem + + def system(self, cmd, environ=None, cwd=None, onerr=None, + errprefix=None): + # fallback to the original system method if the output needs to be + # captured (to self._buffers), or the output stream is not stdout + # (e.g. stderr, cStringIO), because the chg client is not aware of + # these situations and will behave differently (write to stdout). + if (any(s[1] for s in self._bufferstates) + or not util.safehasattr(self.fout, 'fileno') + or self.fout.fileno() != util.stdout.fileno()): + return super(chgui, self).system(cmd, environ, cwd, onerr, + errprefix) + self.flush() + rc = self._csystem(cmd, util.shellenviron(environ), cwd) + if rc and onerr: + errmsg = '%s %s' % (os.path.basename(cmd.split(None, 1)[0]), + util.explainexit(rc)[0]) + if errprefix: + errmsg = '%s: %s' % (errprefix, errmsg) + raise onerr(errmsg) + return rc + + def _runpager(self, cmd): + self._csystem(cmd, util.shellenviron(), type='pager', + cmdtable={'attachio': attachio}) + + return chgui(srcui) + +def _loadnewui(srcui, args): + from . import dispatch # avoid cycle + + newui = srcui.__class__.load() + for a in ['fin', 'fout', 'ferr', 'environ']: + setattr(newui, a, getattr(srcui, a)) + if util.safehasattr(srcui, '_csystem'): + newui._csystem = srcui._csystem + + # command line args + args = args[:] + dispatch._parseconfig(newui, dispatch._earlygetopt(['--config'], args)) + + # stolen from tortoisehg.util.copydynamicconfig() + for section, name, value in srcui.walkconfig(): + source = srcui.configsource(section, name) + if ':' in source or source == '--config': + # path:line or command line + continue + newui.setconfig(section, name, value, source) + + # load wd and repo config, copied from dispatch.py + cwds = dispatch._earlygetopt(['--cwd'], args) + cwd = cwds and os.path.realpath(cwds[-1]) or None + rpath = dispatch._earlygetopt(["-R", "--repository", "--repo"], args) + path, newlui = dispatch._getlocal(newui, rpath, wd=cwd) + + return (newui, newlui) + +class channeledsystem(object): + """Propagate ui.system() request in the following format: + + payload length (unsigned int), + type, '\0', + cmd, '\0', + cwd, '\0', + envkey, '=', val, '\0', + ... + envkey, '=', val + + if type == 'system', waits for: + + exitcode length (unsigned int), + exitcode (int) + + if type == 'pager', repetitively waits for a command name ending with '\n' + and executes it defined by cmdtable, or exits the loop if the command name + is empty. + """ + def __init__(self, in_, out, channel): + self.in_ = in_ + self.out = out + self.channel = channel + + def __call__(self, cmd, environ, cwd=None, type='system', cmdtable=None): + args = [type, util.quotecommand(cmd), os.path.abspath(cwd or '.')] + args.extend('%s=%s' % (k, v) for k, v in environ.iteritems()) + data = '\0'.join(args) + self.out.write(struct.pack('>cI', self.channel, len(data))) + self.out.write(data) + self.out.flush() + + if type == 'system': + length = self.in_.read(4) + length, = struct.unpack('>I', length) + if length != 4: + raise error.Abort(_('invalid response')) + rc, = struct.unpack('>i', self.in_.read(4)) + return rc + elif type == 'pager': + while True: + cmd = self.in_.readline()[:-1] + if not cmd: + break + if cmdtable and cmd in cmdtable: + _log('pager subcommand: %s' % cmd) + cmdtable[cmd]() + else: + raise error.Abort(_('unexpected command: %s') % cmd) + else: + raise error.ProgrammingError('invalid S channel type: %s' % type) + +_iochannels = [ + # server.ch, ui.fp, mode + ('cin', 'fin', 'rb'), + ('cout', 'fout', 'wb'), + ('cerr', 'ferr', 'wb'), +] + +class chgcmdserver(commandserver.server): + def __init__(self, ui, repo, fin, fout, sock, hashstate, baseaddress): + super(chgcmdserver, self).__init__( + _newchgui(ui, channeledsystem(fin, fout, 'S'), self.attachio), + repo, fin, fout) + self.clientsock = sock + self._oldios = [] # original (self.ch, ui.fp, fd) before "attachio" + self.hashstate = hashstate + self.baseaddress = baseaddress + if hashstate is not None: + self.capabilities = self.capabilities.copy() + self.capabilities['validate'] = chgcmdserver.validate + + def cleanup(self): + super(chgcmdserver, self).cleanup() + # dispatch._runcatch() does not flush outputs if exception is not + # handled by dispatch._dispatch() + self.ui.flush() + self._restoreio() + + def attachio(self): + """Attach to client's stdio passed via unix domain socket; all + channels except cresult will no longer be used + """ + # tell client to sendmsg() with 1-byte payload, which makes it + # distinctive from "attachio\n" command consumed by client.read() + self.clientsock.sendall(struct.pack('>cI', 'I', 1)) + clientfds = osutil.recvfds(self.clientsock.fileno()) + _log('received fds: %r\n' % clientfds) + + ui = self.ui + ui.flush() + first = self._saveio() + for fd, (cn, fn, mode) in zip(clientfds, _iochannels): + assert fd > 0 + fp = getattr(ui, fn) + os.dup2(fd, fp.fileno()) + os.close(fd) + if not first: + continue + # reset buffering mode when client is first attached. as we want + # to see output immediately on pager, the mode stays unchanged + # when client re-attached. ferr is unchanged because it should + # be unbuffered no matter if it is a tty or not. + if fn == 'ferr': + newfp = fp + else: + # make it line buffered explicitly because the default is + # decided on first write(), where fout could be a pager. + if fp.isatty(): + bufsize = 1 # line buffered + else: + bufsize = -1 # system default + newfp = os.fdopen(fp.fileno(), mode, bufsize) + setattr(ui, fn, newfp) + setattr(self, cn, newfp) + + self.cresult.write(struct.pack('>i', len(clientfds))) + + def _saveio(self): + if self._oldios: + return False + ui = self.ui + for cn, fn, _mode in _iochannels: + ch = getattr(self, cn) + fp = getattr(ui, fn) + fd = os.dup(fp.fileno()) + self._oldios.append((ch, fp, fd)) + return True + + def _restoreio(self): + ui = self.ui + for (ch, fp, fd), (cn, fn, _mode) in zip(self._oldios, _iochannels): + newfp = getattr(ui, fn) + # close newfp while it's associated with client; otherwise it + # would be closed when newfp is deleted + if newfp is not fp: + newfp.close() + # restore original fd: fp is open again + os.dup2(fd, fp.fileno()) + os.close(fd) + setattr(self, cn, ch) + setattr(ui, fn, fp) + del self._oldios[:] + + def validate(self): + """Reload the config and check if the server is up to date + + Read a list of '\0' separated arguments. + Write a non-empty list of '\0' separated instruction strings or '\0' + if the list is empty. + An instruction string could be either: + - "unlink $path", the client should unlink the path to stop the + outdated server. + - "redirect $path", the client should attempt to connect to $path + first. If it does not work, start a new server. It implies + "reconnect". + - "exit $n", the client should exit directly with code n. + This may happen if we cannot parse the config. + - "reconnect", the client should close the connection and + reconnect. + If neither "reconnect" nor "redirect" is included in the instruction + list, the client can continue with this server after completing all + the instructions. + """ + from . import dispatch # avoid cycle + + args = self._readlist() + try: + self.ui, lui = _loadnewui(self.ui, args) + except error.ParseError as inst: + dispatch._formatparse(self.ui.warn, inst) + self.ui.flush() + self.cresult.write('exit 255') + return + newhash = hashstate.fromui(lui, self.hashstate.mtimepaths) + insts = [] + if newhash.mtimehash != self.hashstate.mtimehash: + addr = _hashaddress(self.baseaddress, self.hashstate.confighash) + insts.append('unlink %s' % addr) + # mtimehash is empty if one or more extensions fail to load. + # to be compatible with hg, still serve the client this time. + if self.hashstate.mtimehash: + insts.append('reconnect') + if newhash.confighash != self.hashstate.confighash: + addr = _hashaddress(self.baseaddress, newhash.confighash) + insts.append('redirect %s' % addr) + _log('validate: %s\n' % insts) + self.cresult.write('\0'.join(insts) or '\0') + + def chdir(self): + """Change current directory + + Note that the behavior of --cwd option is bit different from this. + It does not affect --config parameter. + """ + path = self._readstr() + if not path: + return + _log('chdir to %r\n' % path) + os.chdir(path) + + def setumask(self): + """Change umask""" + mask = struct.unpack('>I', self._read(4))[0] + _log('setumask %r\n' % mask) + os.umask(mask) + + def runcommand(self): + return super(chgcmdserver, self).runcommand() + + def setenv(self): + """Clear and update os.environ + + Note that not all variables can make an effect on the running process. + """ + l = self._readlist() + try: + newenv = dict(s.split('=', 1) for s in l) + except ValueError: + raise ValueError('unexpected value in setenv request') + _log('setenv: %r\n' % sorted(newenv.keys())) + encoding.environ.clear() + encoding.environ.update(newenv) + + capabilities = commandserver.server.capabilities.copy() + capabilities.update({'attachio': attachio, + 'chdir': chdir, + 'runcommand': runcommand, + 'setenv': setenv, + 'setumask': setumask}) + + if util.safehasattr(osutil, 'setprocname'): + def setprocname(self): + """Change process title""" + name = self._readstr() + _log('setprocname: %r\n' % name) + osutil.setprocname(name) + capabilities['setprocname'] = setprocname + +def _tempaddress(address): + return '%s.%d.tmp' % (address, os.getpid()) + +def _hashaddress(address, hashstr): + # if the basename of address contains '.', use only the left part. this + # makes it possible for the client to pass 'server.tmp$PID' and follow by + # an atomic rename to avoid locking when spawning new servers. + dirname, basename = os.path.split(address) + basename = basename.split('.', 1)[0] + return '%s-%s' % (os.path.join(dirname, basename), hashstr) + +class chgunixservicehandler(object): + """Set of operations for chg services""" + + pollinterval = 1 # [sec] + + def __init__(self, ui): + self.ui = ui + self._idletimeout = ui.configint('chgserver', 'idletimeout', 3600) + self._lastactive = time.time() + + def bindsocket(self, sock, address): + self._inithashstate(address) + self._checkextensions() + self._bind(sock) + self._createsymlink() + + def _inithashstate(self, address): + self._baseaddress = address + if self.ui.configbool('chgserver', 'skiphash', False): + self._hashstate = None + self._realaddress = address + return + self._hashstate = hashstate.fromui(self.ui) + self._realaddress = _hashaddress(address, self._hashstate.confighash) + + def _checkextensions(self): + if not self._hashstate: + return + if extensions.notloaded(): + # one or more extensions failed to load. mtimehash becomes + # meaningless because we do not know the paths of those extensions. + # set mtimehash to an illegal hash value to invalidate the server. + self._hashstate.mtimehash = '' + + def _bind(self, sock): + # use a unique temp address so we can stat the file and do ownership + # check later + tempaddress = _tempaddress(self._realaddress) + util.bindunixsocket(sock, tempaddress) + self._socketstat = os.stat(tempaddress) + # rename will replace the old socket file if exists atomically. the + # old server will detect ownership change and exit. + util.rename(tempaddress, self._realaddress) + + def _createsymlink(self): + if self._baseaddress == self._realaddress: + return + tempaddress = _tempaddress(self._baseaddress) + os.symlink(os.path.basename(self._realaddress), tempaddress) + util.rename(tempaddress, self._baseaddress) + + def _issocketowner(self): + try: + stat = os.stat(self._realaddress) + return (stat.st_ino == self._socketstat.st_ino and + stat.st_mtime == self._socketstat.st_mtime) + except OSError: + return False + + def unlinksocket(self, address): + if not self._issocketowner(): + return + # it is possible to have a race condition here that we may + # remove another server's socket file. but that's okay + # since that server will detect and exit automatically and + # the client will start a new server on demand. + try: + os.unlink(self._realaddress) + except OSError as exc: + if exc.errno != errno.ENOENT: + raise + + def printbanner(self, address): + # no "listening at" message should be printed to simulate hg behavior + pass + + def shouldexit(self): + if not self._issocketowner(): + self.ui.debug('%s is not owned, exiting.\n' % self._realaddress) + return True + if time.time() - self._lastactive > self._idletimeout: + self.ui.debug('being idle too long. exiting.\n') + return True + return False + + def newconnection(self): + self._lastactive = time.time() + + def createcmdserver(self, repo, conn, fin, fout): + return chgcmdserver(self.ui, repo, fin, fout, conn, + self._hashstate, self._baseaddress) + +def chgunixservice(ui, repo, opts): + # CHGINTERNALMARK is temporarily set by chg client to detect if chg will + # start another chg. drop it to avoid possible side effects. + if 'CHGINTERNALMARK' in encoding.environ: + del encoding.environ['CHGINTERNALMARK'] + + if repo: + # one chgserver can serve multiple repos. drop repo information + ui.setconfig('bundle', 'mainreporoot', '', 'repo') + h = chgunixservicehandler(ui) + return commandserver.unixforkingservice(ui, repo=None, opts=opts, handler=h) diff -Nru mercurial-4.0.2~trusty1/mercurial/cmdutil.py mercurial-4.1~rc~trusty1/mercurial/cmdutil.py --- mercurial-4.0.2~trusty1/mercurial/cmdutil.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/cmdutil.py 2017-01-18 16:54:51.000000000 +0000 @@ -10,7 +10,6 @@ import errno import os import re -import sys import tempfile from .i18n import _ @@ -27,16 +26,19 @@ changelog, copies, crecord as crecordmod, + dirstateguard as dirstateguardmod, encoding, error, formatter, graphmod, lock as lockmod, match as matchmod, + mergeutil, obsolete, patch, pathutil, phases, + pycompat, repair, revlog, revset, @@ -47,6 +49,10 @@ ) stringio = util.stringio +# special string such that everything below this line will be ingored in the +# editor text +_linebelow = "^HG: ------------------------ >8 ------------------------$" + def ishunk(x): hunkclasses = (crecordmod.uihunk, patch.recordhunk) return isinstance(x, hunkclasses) @@ -83,7 +89,7 @@ else: recordfn = crecordmod.chunkselector - return crecordmod.filterpatch(ui, originalhunks, recordfn) + return crecordmod.filterpatch(ui, originalhunks, recordfn, operation) else: return patch.filterpatch(ui, originalhunks, operation) @@ -225,7 +231,8 @@ + crecordmod.patchhelptext + fp.read()) reviewedpatch = ui.edit(patchtext, "", - extra={"suffix": ".diff"}) + extra={"suffix": ".diff"}, + tmpdir=repo.path) fp.truncate(0) fp.write(reviewedpatch) fp.seek(0) @@ -349,15 +356,23 @@ return p -def bailifchanged(repo, merge=True): +def bailifchanged(repo, merge=True, hint=None): + """ enforce the precondition that working directory must be clean. + + 'merge' can be set to false if a pending uncommitted merge should be + ignored (such as when 'update --check' runs). + + 'hint' is the usual hint given to Abort exception. + """ + if merge and repo.dirstate.p2() != nullid: - raise error.Abort(_('outstanding uncommitted merge')) + raise error.Abort(_('outstanding uncommitted merge'), hint=hint) modified, added, removed, deleted = repo.status()[:4] if modified or added or removed or deleted: - raise error.Abort(_('uncommitted changes')) + raise error.Abort(_('uncommitted changes'), hint=hint) ctx = repo[None] for s in sorted(ctx.substate): - ctx.sub(s).bailifchanged() + ctx.sub(s).bailifchanged(hint=hint) def logmessage(ui, opts): """ get the log message according to -m and -l option """ @@ -555,11 +570,11 @@ if 'treemanifest' not in repo.requirements: raise error.Abort(_("--dir can only be used on repos with " "treemanifest enabled")) - dirlog = repo.manifest.dirlog(dir) + dirlog = repo.manifestlog._revlog.dirlog(dir) if len(dirlog): r = dirlog elif mf: - r = repo.manifest + r = repo.manifestlog._revlog elif file_: filelog = repo.file(file_) if len(filelog): @@ -569,7 +584,7 @@ raise error.CommandError(cmd, _('invalid arguments')) if not os.path.isfile(file_): raise error.Abort(_("revlog '%s' not found") % file_) - r = revlog.revlog(scmutil.opener(os.getcwd(), audit=False), + r = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), file_[:-2] + ".i") return r @@ -729,7 +744,7 @@ else: striplen = len(abspfx) if striplen: - striplen += len(os.sep) + striplen += len(pycompat.ossep) res = lambda p: os.path.join(dest, util.localpath(p)[striplen:]) elif destdirexists: res = lambda p: os.path.join(dest, @@ -763,12 +778,12 @@ abspfx = util.localpath(abspfx) striplen = len(abspfx) if striplen: - striplen += len(os.sep) + striplen += len(pycompat.ossep) if os.path.isdir(os.path.join(dest, os.path.split(abspfx)[1])): score = evalpath(striplen) striplen1 = len(os.path.split(abspfx)[0]) if striplen1: - striplen1 += len(os.sep) + striplen1 += len(pycompat.ossep) if evalpath(striplen1) > score: striplen = striplen1 res = lambda p: os.path.join(dest, @@ -819,93 +834,6 @@ return errors != 0 -def service(opts, parentfn=None, initfn=None, runfn=None, logfile=None, - runargs=None, appendpid=False): - '''Run a command as a service.''' - - def writepid(pid): - if opts['pid_file']: - if appendpid: - mode = 'a' - else: - mode = 'w' - fp = open(opts['pid_file'], mode) - fp.write(str(pid) + '\n') - fp.close() - - if opts['daemon'] and not opts['daemon_postexec']: - # Signal child process startup with file removal - lockfd, lockpath = tempfile.mkstemp(prefix='hg-service-') - os.close(lockfd) - try: - if not runargs: - runargs = util.hgcmd() + sys.argv[1:] - runargs.append('--daemon-postexec=unlink:%s' % lockpath) - # Don't pass --cwd to the child process, because we've already - # changed directory. - for i in xrange(1, len(runargs)): - if runargs[i].startswith('--cwd='): - del runargs[i] - break - elif runargs[i].startswith('--cwd'): - del runargs[i:i + 2] - break - def condfn(): - return not os.path.exists(lockpath) - pid = util.rundetached(runargs, condfn) - if pid < 0: - raise error.Abort(_('child process failed to start')) - writepid(pid) - finally: - try: - os.unlink(lockpath) - except OSError as e: - if e.errno != errno.ENOENT: - raise - if parentfn: - return parentfn(pid) - else: - return - - if initfn: - initfn() - - if not opts['daemon']: - writepid(util.getpid()) - - if opts['daemon_postexec']: - try: - os.setsid() - except AttributeError: - pass - for inst in opts['daemon_postexec']: - if inst.startswith('unlink:'): - lockpath = inst[7:] - os.unlink(lockpath) - elif inst.startswith('chdir:'): - os.chdir(inst[6:]) - elif inst != 'none': - raise error.Abort(_('invalid value for --daemon-postexec: %s') - % inst) - util.hidewindow() - sys.stdout.flush() - sys.stderr.flush() - - nullfd = os.open(os.devnull, os.O_RDWR) - logfilefd = nullfd - if logfile: - logfilefd = os.open(logfile, os.O_RDWR | os.O_CREAT | os.O_APPEND) - os.dup2(nullfd, 0) - os.dup2(logfilefd, 1) - os.dup2(logfilefd, 2) - if nullfd not in (0, 1, 2): - os.close(nullfd) - if logfile and logfilefd not in (0, 1, 2): - os.close(logfilefd) - - if runfn: - return runfn() - ## facility to let extension process additional data into an import patch # list of identifier to be executed in order extrapreimport = [] # run before commit @@ -1202,8 +1130,7 @@ chunks = patch.diff(repo, node1, node2, match, changes, diffopts, prefix=prefix, relroot=relroot) for chunk, label in patch.diffstatui(util.iterlines(chunks), - width=width, - git=diffopts.git): + width=width): write(chunk, label=label) else: for chunk, label in patch.diffui(repo, node1, node2, match, @@ -1228,6 +1155,14 @@ sub.diff(ui, diffopts, tempnode2, submatch, changes=changes, stat=stat, fp=fp, prefix=prefix) +def _changesetlabels(ctx): + labels = ['log.changeset', 'changeset.%s' % ctx.phasestr()] + if ctx.troubled(): + labels.append('changeset.troubled') + for trouble in ctx.troubles(): + labels.append('trouble.%s' % trouble) + return ' '.join(labels) + class changeset_printer(object): '''show changeset information when templating not requested.''' @@ -1288,7 +1223,7 @@ # i18n: column positioning for "hg log" self.ui.write(_("changeset: %d:%s\n") % revnode, - label='log.changeset changeset.%s' % ctx.phasestr()) + label=_changesetlabels(ctx)) # branches are shown first before any other names due to backwards # compatibility @@ -1324,7 +1259,8 @@ mnode = ctx.manifestnode() # i18n: column positioning for "hg log" self.ui.write(_("manifest: %d:%s\n") % - (self.repo.manifest.rev(mnode), hex(mnode)), + (self.repo.manifestlog._revlog.rev(mnode), + hex(mnode)), label='ui.debug log.manifest') # i18n: column positioning for "hg log" self.ui.write(_("user: %s\n") % ctx.user(), @@ -1333,6 +1269,11 @@ self.ui.write(_("date: %s\n") % date, label='log.date') + if ctx.troubled(): + # i18n: column positioning for "hg log" + self.ui.write(_("trouble: %s\n") % ', '.join(ctx.troubles()), + label='log.trouble') + if self.ui.debugflag: files = ctx.p1().status(ctx)[:3] for key, value in zip([# i18n: column positioning for "hg log" @@ -1508,6 +1449,7 @@ 'parent': '{rev}:{node|formatnode} ', 'manifest': '{rev}:{node|formatnode}', 'file_copy': '{name} ({source})', + 'envvar': '{key}={value}', 'extra': '{key}={value|stringescape}' } # filecopy is preserved for compatibility reasons @@ -2566,11 +2508,14 @@ # for performance to avoid the cost of parsing the manifest. if len(matcher.files()) == 1 and not matcher.anypats(): file = matcher.files()[0] - mf = repo.manifest + mfl = repo.manifestlog mfnode = ctx.manifestnode() - if mfnode and mf.find(mfnode, file)[0]: - write(file) - return 0 + try: + if mfnode and mfl[mfnode].find(file)[0]: + write(file) + return 0 + except KeyError: + pass for abs in ctx.walk(matcher): write(abs) @@ -2827,7 +2772,7 @@ committext = buildcommittext(repo, ctx, subs, extramsg) # run editor in the repository root - olddir = os.getcwd() + olddir = pycompat.getcwd() os.chdir(repo.root) # make in-memory changes visible to external process @@ -2836,8 +2781,17 @@ pending = tr and tr.writepending() and repo.root editortext = repo.ui.edit(committext, ctx.user(), ctx.extra(), - editform=editform, pending=pending) - text = re.sub("(?m)^HG:.*(\n|$)", "", editortext) + editform=editform, pending=pending, + tmpdir=repo.path) + text = editortext + + # strip away anything below this special string (used for editors that want + # to display the diff) + stripbelow = re.search(_linebelow, text, flags=re.MULTILINE) + if stripbelow: + text = text[:stripbelow.start()] + + text = re.sub("(?m)^HG:.*(\n|$)", "", text) os.chdir(olddir) if finishdesc: @@ -3249,13 +3203,18 @@ fc = ctx[f] repo.wwrite(f, fc.data(), fc.flags()) + def doremove(f): + try: + util.unlinkpath(repo.wjoin(f)) + except OSError: + pass + repo.dirstate.remove(f) + audit_path = pathutil.pathauditor(repo.root) for f in actions['forget'][0]: if interactive: - choice = \ - repo.ui.promptchoice( - _("forget added file %s (yn)?$$ &Yes $$ &No") - % f) + choice = repo.ui.promptchoice( + _("forget added file %s (Yn)?$$ &Yes $$ &No") % f) if choice == 0: repo.dirstate.drop(f) else: @@ -3264,11 +3223,15 @@ repo.dirstate.drop(f) for f in actions['remove'][0]: audit_path(f) - try: - util.unlinkpath(repo.wjoin(f)) - except OSError: - pass - repo.dirstate.remove(f) + if interactive: + choice = repo.ui.promptchoice( + _("remove added file %s (Yn)?$$ &Yes $$ &No") % f) + if choice == 0: + doremove(f) + else: + excluded_files.append(repo.wjoin(f)) + else: + doremove(f) for f in actions['drop'][0]: audit_path(f) repo.dirstate.remove(f) @@ -3403,6 +3366,11 @@ return cmd +def checkunresolved(ms): + ms._repo.ui.deprecwarn('checkunresolved moved from cmdutil to mergeutil', + '4.1') + return mergeutil.checkunresolved(ms) + # a list of (ui, repo, otherpeer, opts, missing) functions called by # commands.outgoing. "missing" is "missing" of the result of # "findcommonoutgoing()" @@ -3463,7 +3431,7 @@ '''Check for an unfinished operation and return the command to finish it. - afterresolvedstates tupples define a .hg/{file} and the corresponding + afterresolvedstates tuples define a .hg/{file} and the corresponding command needed to finish it. Returns a (msg, warning) tuple. 'msg' is a string and 'warning' is @@ -3510,57 +3478,9 @@ hint = after[0] raise error.Abort(_('no %s in progress') % task, hint=hint) -class dirstateguard(object): - '''Restore dirstate at unexpected failure. - - At the construction, this class does: - - - write current ``repo.dirstate`` out, and - - save ``.hg/dirstate`` into the backup file - - This restores ``.hg/dirstate`` from backup file, if ``release()`` - is invoked before ``close()``. - - This just removes the backup file at ``close()`` before ``release()``. - ''' - +class dirstateguard(dirstateguardmod.dirstateguard): def __init__(self, repo, name): - self._repo = repo - self._active = False - self._closed = False - self._suffix = '.backup.%s.%d' % (name, id(self)) - repo.dirstate.savebackup(repo.currenttransaction(), self._suffix) - self._active = True - - def __del__(self): - if self._active: # still active - # this may occur, even if this class is used correctly: - # for example, releasing other resources like transaction - # may raise exception before ``dirstateguard.release`` in - # ``release(tr, ....)``. - self._abort() - - def close(self): - if not self._active: # already inactivated - msg = (_("can't close already inactivated backup: dirstate%s") - % self._suffix) - raise error.Abort(msg) - - self._repo.dirstate.clearbackup(self._repo.currenttransaction(), - self._suffix) - self._active = False - self._closed = True - - def _abort(self): - self._repo.dirstate.restorebackup(self._repo.currenttransaction(), - self._suffix) - self._active = False - - def release(self): - if not self._closed: - if not self._active: # already inactivated - msg = (_("can't release already inactivated backup:" - " dirstate%s") - % self._suffix) - raise error.Abort(msg) - self._abort() + dirstateguardmod.dirstateguard.__init__(self, repo, name) + repo.ui.deprecwarn( + 'dirstateguard has moved from cmdutil to dirstateguard', + '4.1') diff -Nru mercurial-4.0.2~trusty1/mercurial/color.py mercurial-4.1~rc~trusty1/mercurial/color.py --- mercurial-4.0.2~trusty1/mercurial/color.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/color.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,65 @@ +# utility for color output for Mercurial commands +# +# Copyright (C) 2007 Kevin Christen and other +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +_styles = {'grep.match': 'red bold', + 'grep.linenumber': 'green', + 'grep.rev': 'green', + 'grep.change': 'green', + 'grep.sep': 'cyan', + 'grep.filename': 'magenta', + 'grep.user': 'magenta', + 'grep.date': 'magenta', + 'bookmarks.active': 'green', + 'branches.active': 'none', + 'branches.closed': 'black bold', + 'branches.current': 'green', + 'branches.inactive': 'none', + 'diff.changed': 'white', + 'diff.deleted': 'red', + 'diff.diffline': 'bold', + 'diff.extended': 'cyan bold', + 'diff.file_a': 'red bold', + 'diff.file_b': 'green bold', + 'diff.hunk': 'magenta', + 'diff.inserted': 'green', + 'diff.tab': '', + 'diff.trailingwhitespace': 'bold red_background', + 'changeset.public' : '', + 'changeset.draft' : '', + 'changeset.secret' : '', + 'diffstat.deleted': 'red', + 'diffstat.inserted': 'green', + 'histedit.remaining': 'red bold', + 'ui.prompt': 'yellow', + 'log.changeset': 'yellow', + 'patchbomb.finalsummary': '', + 'patchbomb.from': 'magenta', + 'patchbomb.to': 'cyan', + 'patchbomb.subject': 'green', + 'patchbomb.diffstats': '', + 'rebase.rebased': 'blue', + 'rebase.remaining': 'red bold', + 'resolve.resolved': 'green bold', + 'resolve.unresolved': 'red bold', + 'shelve.age': 'cyan', + 'shelve.newest': 'green bold', + 'shelve.name': 'blue bold', + 'status.added': 'green bold', + 'status.clean': 'none', + 'status.copied': 'none', + 'status.deleted': 'cyan bold underline', + 'status.ignored': 'black bold', + 'status.modified': 'blue bold', + 'status.removed': 'red bold', + 'status.unknown': 'magenta bold underline', + 'tags.normal': 'green', + 'tags.local': 'black bold'} + +def loadcolortable(ui, extname, colortable): + _styles.update(colortable) diff -Nru mercurial-4.0.2~trusty1/mercurial/commandserver.py mercurial-4.1~rc~trusty1/mercurial/commandserver.py --- mercurial-4.0.2~trusty1/mercurial/commandserver.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/commandserver.py 2017-01-18 16:54:51.000000000 +0000 @@ -15,13 +15,13 @@ import signal import socket import struct -import sys import traceback from .i18n import _ from . import ( encoding, error, + pycompat, util, ) @@ -54,8 +54,8 @@ def write(self, data): if not data: return - self.out.write(struct.pack('>cI', self.channel, len(data))) - self.out.write(data) + # single write() to guarantee the same atomicity as the underlying file + self.out.write(struct.pack('>cI', self.channel, len(data)) + data) self.out.flush() def __getattr__(self, attr): @@ -153,7 +153,7 @@ based stream to fout. """ def __init__(self, ui, repo, fin, fout): - self.cwd = os.getcwd() + self.cwd = pycompat.getcwd() # developer config: cmdserver.log logpath = ui.config("cmdserver", "log", None) @@ -304,8 +304,8 @@ ui.flush() newfiles = [] nullfd = os.open(os.devnull, os.O_RDWR) - for f, sysf, mode in [(ui.fin, sys.stdin, 'rb'), - (ui.fout, sys.stdout, 'wb')]: + for f, sysf, mode in [(ui.fin, util.stdin, 'rb'), + (ui.fout, util.stdout, 'wb')]: if f is sysf: newfd = os.dup(f.fileno()) os.dup2(nullfd, f.fileno()) @@ -530,15 +530,3 @@ _serverequest(self.ui, self.repo, conn, h.createcmdserver) finally: gc.collect() # trigger __del__ since worker process uses os._exit - -_servicemap = { - 'pipe': pipeservice, - 'unix': unixforkingservice, - } - -def createservice(ui, repo, opts): - mode = opts['cmdserver'] - try: - return _servicemap[mode](ui, repo, opts) - except KeyError: - raise error.Abort(_('unknown mode %s') % mode) diff -Nru mercurial-4.0.2~trusty1/mercurial/commands.py mercurial-4.1~rc~trusty1/mercurial/commands.py --- mercurial-4.0.2~trusty1/mercurial/commands.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/commands.py 2017-01-18 16:54:51.000000000 +0000 @@ -9,12 +9,10 @@ import difflib import errno -import operator import os -import random import re -import shlex import socket +import string import sys import tempfile import time @@ -34,25 +32,19 @@ bundle2, changegroup, cmdutil, - commandserver, - context, copies, - dagparser, - dagutil, destutil, + dirstateguard, discovery, encoding, error, exchange, extensions, - fileset, formatter, graphmod, hbisect, help, hg, - hgweb, - localrepo, lock as lockmod, merge as mergemod, minirst, @@ -61,18 +53,17 @@ phases, policy, pvec, + pycompat, repair, revlog, revset, scmutil, - setdiscovery, - simplemerge, + server, sshserver, sslutil, streamclone, templatekw, templater, - treediscovery, ui as uimod, util, ) @@ -686,7 +677,7 @@ bheads = repo.branchheads(branch) rctx = scmutil.revsingle(repo, hex(parent)) if not opts.get('merge') and op1 != node: - dsguard = cmdutil.dirstateguard(repo, 'backout') + dsguard = dirstateguard.dirstateguard(repo, 'backout') try: ui.setconfig('ui', 'forcemerge', opts.get('tool', ''), 'backout') @@ -832,7 +823,7 @@ hg log --graph -r "bisect(range)" - See :hg:`help revsets` for more about the `bisect()` keyword. + See :hg:`help revisions.bisect` for more about the `bisect()` predicate. Returns 0 on success. """ @@ -1392,7 +1383,17 @@ assert cgversion == '02' bversion = 'HG20' - bundle2.writebundle(ui, cg, fname, bversion, compression=bcompression) + # TODO compression options should be derived from bundlespec parsing. + # This is a temporary hack to allow adjusting bundle compression + # level without a) formalizing the bundlespec changes to declare it + # b) introducing a command flag. + compopts = {} + complevel = ui.configint('experimental', 'bundlecomplevel') + if complevel is not None: + compopts['level'] = complevel + + bundle2.writebundle(ui, cg, fname, bversion, compression=bcompression, + compopts=compopts) @command('cat', [('o', 'output', '', @@ -1812,29 +1813,28 @@ raise error.Abort(_('only one config item permitted')) matched = False for section, name, value in ui.walkconfig(untrusted=untrusted): + source = ui.configsource(section, name, untrusted) value = str(value) if fm.isplain(): + source = source or 'none' value = value.replace('\n', '\\n') entryname = section + '.' + name if values: for v in values: if v == section: fm.startitem() - fm.condwrite(ui.debugflag, 'source', '%s: ', - ui.configsource(section, name, untrusted)) + fm.condwrite(ui.debugflag, 'source', '%s: ', source) fm.write('name value', '%s=%s\n', entryname, value) matched = True elif v == entryname: fm.startitem() - fm.condwrite(ui.debugflag, 'source', '%s: ', - ui.configsource(section, name, untrusted)) + fm.condwrite(ui.debugflag, 'source', '%s: ', source) fm.write('value', '%s\n', value) fm.data(name=entryname) matched = True else: fm.startitem() - fm.condwrite(ui.debugflag, 'source', '%s: ', - ui.configsource(section, name, untrusted)) + fm.condwrite(ui.debugflag, 'source', '%s: ', source) fm.write('name value', '%s=%s\n', entryname, value) matched = True fm.end() @@ -1866,806 +1866,6 @@ with repo.wlock(False): return cmdutil.copy(ui, repo, pats, opts) -@command('debugancestor', [], _('[INDEX] REV1 REV2'), optionalrepo=True) -def debugancestor(ui, repo, *args): - """find the ancestor revision of two revisions in a given index""" - if len(args) == 3: - index, rev1, rev2 = args - r = revlog.revlog(scmutil.opener(os.getcwd(), audit=False), index) - lookup = r.lookup - elif len(args) == 2: - if not repo: - raise error.Abort(_("there is no Mercurial repository here " - "(.hg not found)")) - rev1, rev2 = args - r = repo.changelog - lookup = repo.lookup - else: - raise error.Abort(_('either two or three arguments required')) - a = r.ancestor(lookup(rev1), lookup(rev2)) - ui.write("%d:%s\n" % (r.rev(a), hex(a))) - -@command('debugbuilddag', - [('m', 'mergeable-file', None, _('add single file mergeable changes')), - ('o', 'overwritten-file', None, _('add single file all revs overwrite')), - ('n', 'new-file', None, _('add new file at each rev'))], - _('[OPTION]... [TEXT]')) -def debugbuilddag(ui, repo, text=None, - mergeable_file=False, - overwritten_file=False, - new_file=False): - """builds a repo with a given DAG from scratch in the current empty repo - - The description of the DAG is read from stdin if not given on the - command line. - - Elements: - - - "+n" is a linear run of n nodes based on the current default parent - - "." is a single node based on the current default parent - - "$" resets the default parent to null (implied at the start); - otherwise the default parent is always the last node created - - " 0: - raise error.Abort(_('repository is not empty')) - - # determine number of revs in DAG - total = 0 - for type, data in dagparser.parsedag(text): - if type == 'n': - total += 1 - - if mergeable_file: - linesperrev = 2 - # make a file with k lines per rev - initialmergedlines = [str(i) for i in xrange(0, total * linesperrev)] - initialmergedlines.append("") - - tags = [] - - wlock = lock = tr = None - try: - wlock = repo.wlock() - lock = repo.lock() - tr = repo.transaction("builddag") - - at = -1 - atbranch = 'default' - nodeids = [] - id = 0 - ui.progress(_('building'), id, unit=_('revisions'), total=total) - for type, data in dagparser.parsedag(text): - if type == 'n': - ui.note(('node %s\n' % str(data))) - id, ps = data - - files = [] - fctxs = {} - - p2 = None - if mergeable_file: - fn = "mf" - p1 = repo[ps[0]] - if len(ps) > 1: - p2 = repo[ps[1]] - pa = p1.ancestor(p2) - base, local, other = [x[fn].data() for x in (pa, p1, - p2)] - m3 = simplemerge.Merge3Text(base, local, other) - ml = [l.strip() for l in m3.merge_lines()] - ml.append("") - elif at > 0: - ml = p1[fn].data().split("\n") - else: - ml = initialmergedlines - ml[id * linesperrev] += " r%i" % id - mergedtext = "\n".join(ml) - files.append(fn) - fctxs[fn] = context.memfilectx(repo, fn, mergedtext) - - if overwritten_file: - fn = "of" - files.append(fn) - fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) - - if new_file: - fn = "nf%i" % id - files.append(fn) - fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) - if len(ps) > 1: - if not p2: - p2 = repo[ps[1]] - for fn in p2: - if fn.startswith("nf"): - files.append(fn) - fctxs[fn] = p2[fn] - - def fctxfn(repo, cx, path): - return fctxs.get(path) - - if len(ps) == 0 or ps[0] < 0: - pars = [None, None] - elif len(ps) == 1: - pars = [nodeids[ps[0]], None] - else: - pars = [nodeids[p] for p in ps] - cx = context.memctx(repo, pars, "r%i" % id, files, fctxfn, - date=(id, 0), - user="debugbuilddag", - extra={'branch': atbranch}) - nodeid = repo.commitctx(cx) - nodeids.append(nodeid) - at = id - elif type == 'l': - id, name = data - ui.note(('tag %s\n' % name)) - tags.append("%s %s\n" % (hex(repo.changelog.node(id)), name)) - elif type == 'a': - ui.note(('branch %s\n' % data)) - atbranch = data - ui.progress(_('building'), id, unit=_('revisions'), total=total) - tr.close() - - if tags: - repo.vfs.write("localtags", "".join(tags)) - finally: - ui.progress(_('building'), None) - release(tr, lock, wlock) - -@command('debugbundle', - [('a', 'all', None, _('show all details')), - ('', 'spec', None, _('print the bundlespec of the bundle'))], - _('FILE'), - norepo=True) -def debugbundle(ui, bundlepath, all=None, spec=None, **opts): - """lists the contents of a bundle""" - with hg.openpath(ui, bundlepath) as f: - if spec: - spec = exchange.getbundlespec(ui, f) - ui.write('%s\n' % spec) - return - - gen = exchange.readbundle(ui, f, bundlepath) - if isinstance(gen, bundle2.unbundle20): - return _debugbundle2(ui, gen, all=all, **opts) - _debugchangegroup(ui, gen, all=all, **opts) - -def _debugchangegroup(ui, gen, all=None, indent=0, **opts): - indent_string = ' ' * indent - if all: - ui.write(("%sformat: id, p1, p2, cset, delta base, len(delta)\n") - % indent_string) - - def showchunks(named): - ui.write("\n%s%s\n" % (indent_string, named)) - chain = None - for chunkdata in iter(lambda: gen.deltachunk(chain), {}): - node = chunkdata['node'] - p1 = chunkdata['p1'] - p2 = chunkdata['p2'] - cs = chunkdata['cs'] - deltabase = chunkdata['deltabase'] - delta = chunkdata['delta'] - ui.write("%s%s %s %s %s %s %s\n" % - (indent_string, hex(node), hex(p1), hex(p2), - hex(cs), hex(deltabase), len(delta))) - chain = node - - chunkdata = gen.changelogheader() - showchunks("changelog") - chunkdata = gen.manifestheader() - showchunks("manifest") - for chunkdata in iter(gen.filelogheader, {}): - fname = chunkdata['filename'] - showchunks(fname) - else: - if isinstance(gen, bundle2.unbundle20): - raise error.Abort(_('use debugbundle2 for this file')) - chunkdata = gen.changelogheader() - chain = None - for chunkdata in iter(lambda: gen.deltachunk(chain), {}): - node = chunkdata['node'] - ui.write("%s%s\n" % (indent_string, hex(node))) - chain = node - -def _debugbundle2(ui, gen, all=None, **opts): - """lists the contents of a bundle2""" - if not isinstance(gen, bundle2.unbundle20): - raise error.Abort(_('not a bundle2 file')) - ui.write(('Stream params: %s\n' % repr(gen.params))) - for part in gen.iterparts(): - ui.write('%s -- %r\n' % (part.type, repr(part.params))) - if part.type == 'changegroup': - version = part.params.get('version', '01') - cg = changegroup.getunbundler(version, part, 'UN') - _debugchangegroup(ui, cg, all=all, indent=4, **opts) - -@command('debugcreatestreamclonebundle', [], 'FILE') -def debugcreatestreamclonebundle(ui, repo, fname): - """create a stream clone bundle file - - Stream bundles are special bundles that are essentially archives of - revlog files. They are commonly used for cloning very quickly. - """ - requirements, gen = streamclone.generatebundlev1(repo) - changegroup.writechunks(ui, gen, fname) - - ui.write(_('bundle requirements: %s\n') % ', '.join(sorted(requirements))) - -@command('debugapplystreamclonebundle', [], 'FILE') -def debugapplystreamclonebundle(ui, repo, fname): - """apply a stream clone bundle file""" - f = hg.openpath(ui, fname) - gen = exchange.readbundle(ui, f, fname) - gen.apply(repo) - -@command('debugcheckstate', [], '') -def debugcheckstate(ui, repo): - """validate the correctness of the current dirstate""" - parent1, parent2 = repo.dirstate.parents() - m1 = repo[parent1].manifest() - m2 = repo[parent2].manifest() - errors = 0 - for f in repo.dirstate: - state = repo.dirstate[f] - if state in "nr" and f not in m1: - ui.warn(_("%s in state %s, but not in manifest1\n") % (f, state)) - errors += 1 - if state in "a" and f in m1: - ui.warn(_("%s in state %s, but also in manifest1\n") % (f, state)) - errors += 1 - if state in "m" and f not in m1 and f not in m2: - ui.warn(_("%s in state %s, but not in either manifest\n") % - (f, state)) - errors += 1 - for f in m1: - state = repo.dirstate[f] - if state not in "nrm": - ui.warn(_("%s in manifest1, but listed as state %s") % (f, state)) - errors += 1 - if errors: - error = _(".hg/dirstate inconsistent with current parent's manifest") - raise error.Abort(error) - -@command('debugcommands', [], _('[COMMAND]'), norepo=True) -def debugcommands(ui, cmd='', *args): - """list all available commands and options""" - for cmd, vals in sorted(table.iteritems()): - cmd = cmd.split('|')[0].strip('^') - opts = ', '.join([i[1] for i in vals[1]]) - ui.write('%s: %s\n' % (cmd, opts)) - -@command('debugcomplete', - [('o', 'options', None, _('show the command options'))], - _('[-o] CMD'), - norepo=True) -def debugcomplete(ui, cmd='', **opts): - """returns the completion list associated with the given command""" - - if opts.get('options'): - options = [] - otables = [globalopts] - if cmd: - aliases, entry = cmdutil.findcmd(cmd, table, False) - otables.append(entry[1]) - for t in otables: - for o in t: - if "(DEPRECATED)" in o[3]: - continue - if o[0]: - options.append('-%s' % o[0]) - options.append('--%s' % o[1]) - ui.write("%s\n" % "\n".join(options)) - return - - cmdlist, unused_allcmds = cmdutil.findpossible(cmd, table) - if ui.verbose: - cmdlist = [' '.join(c[0]) for c in cmdlist.values()] - ui.write("%s\n" % "\n".join(sorted(cmdlist))) - -@command('debugdag', - [('t', 'tags', None, _('use tags as labels')), - ('b', 'branches', None, _('annotate with branch names')), - ('', 'dots', None, _('use dots for runs')), - ('s', 'spaces', None, _('separate elements by spaces'))], - _('[OPTION]... [FILE [REV]...]'), - optionalrepo=True) -def debugdag(ui, repo, file_=None, *revs, **opts): - """format the changelog or an index DAG as a concise textual description - - If you pass a revlog index, the revlog's DAG is emitted. If you list - revision numbers, they get labeled in the output as rN. - - Otherwise, the changelog DAG of the current repo is emitted. - """ - spaces = opts.get('spaces') - dots = opts.get('dots') - if file_: - rlog = revlog.revlog(scmutil.opener(os.getcwd(), audit=False), file_) - revs = set((int(r) for r in revs)) - def events(): - for r in rlog: - yield 'n', (r, list(p for p in rlog.parentrevs(r) - if p != -1)) - if r in revs: - yield 'l', (r, "r%i" % r) - elif repo: - cl = repo.changelog - tags = opts.get('tags') - branches = opts.get('branches') - if tags: - labels = {} - for l, n in repo.tags().items(): - labels.setdefault(cl.rev(n), []).append(l) - def events(): - b = "default" - for r in cl: - if branches: - newb = cl.read(cl.node(r))[5]['branch'] - if newb != b: - yield 'a', newb - b = newb - yield 'n', (r, list(p for p in cl.parentrevs(r) - if p != -1)) - if tags: - ls = labels.get(r) - if ls: - for l in ls: - yield 'l', (r, l) - else: - raise error.Abort(_('need repo for changelog dag')) - - for line in dagparser.dagtextlines(events(), - addspaces=spaces, - wraplabels=True, - wrapannotations=True, - wrapnonlinear=dots, - usedots=dots, - maxlinewidth=70): - ui.write(line) - ui.write("\n") - -@command('debugdata', debugrevlogopts, _('-c|-m|FILE REV')) -def debugdata(ui, repo, file_, rev=None, **opts): - """dump the contents of a data file revision""" - if opts.get('changelog') or opts.get('manifest') or opts.get('dir'): - if rev is not None: - raise error.CommandError('debugdata', _('invalid arguments')) - file_, rev = None, file_ - elif rev is None: - raise error.CommandError('debugdata', _('invalid arguments')) - r = cmdutil.openrevlog(repo, 'debugdata', file_, opts) - try: - ui.write(r.revision(r.lookup(rev))) - except KeyError: - raise error.Abort(_('invalid revision identifier %s') % rev) - -@command('debugdate', - [('e', 'extended', None, _('try extended date formats'))], - _('[-e] DATE [RANGE]'), - norepo=True, optionalrepo=True) -def debugdate(ui, date, range=None, **opts): - """parse and display a date""" - if opts["extended"]: - d = util.parsedate(date, util.extendeddateformats) - else: - d = util.parsedate(date) - ui.write(("internal: %s %s\n") % d) - ui.write(("standard: %s\n") % util.datestr(d)) - if range: - m = util.matchdate(range) - ui.write(("match: %s\n") % m(d[0])) - -@command('debugdiscovery', - [('', 'old', None, _('use old-style discovery')), - ('', 'nonheads', None, - _('use old-style discovery with non-heads included')), - ] + remoteopts, - _('[-l REV] [-r REV] [-b BRANCH]... [OTHER]')) -def debugdiscovery(ui, repo, remoteurl="default", **opts): - """runs the changeset discovery protocol in isolation""" - remoteurl, branches = hg.parseurl(ui.expandpath(remoteurl), - opts.get('branch')) - remote = hg.peer(repo, opts, remoteurl) - ui.status(_('comparing with %s\n') % util.hidepassword(remoteurl)) - - # make sure tests are repeatable - random.seed(12323) - - def doit(localheads, remoteheads, remote=remote): - if opts.get('old'): - if localheads: - raise error.Abort('cannot use localheads with old style ' - 'discovery') - if not util.safehasattr(remote, 'branches'): - # enable in-client legacy support - remote = localrepo.locallegacypeer(remote.local()) - common, _in, hds = treediscovery.findcommonincoming(repo, remote, - force=True) - common = set(common) - if not opts.get('nonheads'): - ui.write(("unpruned common: %s\n") % - " ".join(sorted(short(n) for n in common))) - dag = dagutil.revlogdag(repo.changelog) - all = dag.ancestorset(dag.internalizeall(common)) - common = dag.externalizeall(dag.headsetofconnecteds(all)) - else: - common, any, hds = setdiscovery.findcommonheads(ui, repo, remote) - common = set(common) - rheads = set(hds) - lheads = set(repo.heads()) - ui.write(("common heads: %s\n") % - " ".join(sorted(short(n) for n in common))) - if lheads <= common: - ui.write(("local is subset\n")) - elif rheads <= common: - ui.write(("remote is subset\n")) - - serverlogs = opts.get('serverlog') - if serverlogs: - for filename in serverlogs: - with open(filename, 'r') as logfile: - line = logfile.readline() - while line: - parts = line.strip().split(';') - op = parts[1] - if op == 'cg': - pass - elif op == 'cgss': - doit(parts[2].split(' '), parts[3].split(' ')) - elif op == 'unb': - doit(parts[3].split(' '), parts[2].split(' ')) - line = logfile.readline() - else: - remoterevs, _checkout = hg.addbranchrevs(repo, remote, branches, - opts.get('remote_head')) - localrevs = opts.get('local_head') - doit(localrevs, remoterevs) - -@command('debugextensions', formatteropts, [], norepo=True) -def debugextensions(ui, **opts): - '''show information about active extensions''' - exts = extensions.extensions(ui) - hgver = util.version() - fm = ui.formatter('debugextensions', opts) - for extname, extmod in sorted(exts, key=operator.itemgetter(0)): - isinternal = extensions.ismoduleinternal(extmod) - extsource = extmod.__file__ - if isinternal: - exttestedwith = [] # never expose magic string to users - else: - exttestedwith = getattr(extmod, 'testedwith', '').split() - extbuglink = getattr(extmod, 'buglink', None) - - fm.startitem() - - if ui.quiet or ui.verbose: - fm.write('name', '%s\n', extname) - else: - fm.write('name', '%s', extname) - if isinternal or hgver in exttestedwith: - fm.plain('\n') - elif not exttestedwith: - fm.plain(_(' (untested!)\n')) - else: - lasttestedversion = exttestedwith[-1] - fm.plain(' (%s!)\n' % lasttestedversion) - - fm.condwrite(ui.verbose and extsource, 'source', - _(' location: %s\n'), extsource or "") - - if ui.verbose: - fm.plain(_(' bundled: %s\n') % ['no', 'yes'][isinternal]) - fm.data(bundled=isinternal) - - fm.condwrite(ui.verbose and exttestedwith, 'testedwith', - _(' tested with: %s\n'), - fm.formatlist(exttestedwith, name='ver')) - - fm.condwrite(ui.verbose and extbuglink, 'buglink', - _(' bug reporting: %s\n'), extbuglink or "") - - fm.end() - -@command('debugfileset', - [('r', 'rev', '', _('apply the filespec on this revision'), _('REV'))], - _('[-r REV] FILESPEC')) -def debugfileset(ui, repo, expr, **opts): - '''parse and apply a fileset specification''' - ctx = scmutil.revsingle(repo, opts.get('rev'), None) - if ui.verbose: - tree = fileset.parse(expr) - ui.note(fileset.prettyformat(tree), "\n") - - for f in ctx.getfileset(expr): - ui.write("%s\n" % f) - -@command('debugfsinfo', [], _('[PATH]'), norepo=True) -def debugfsinfo(ui, path="."): - """show information detected about current filesystem""" - util.writefile('.debugfsinfo', '') - ui.write(('exec: %s\n') % (util.checkexec(path) and 'yes' or 'no')) - ui.write(('symlink: %s\n') % (util.checklink(path) and 'yes' or 'no')) - ui.write(('hardlink: %s\n') % (util.checknlink(path) and 'yes' or 'no')) - ui.write(('case-sensitive: %s\n') % (util.fscasesensitive('.debugfsinfo') - and 'yes' or 'no')) - os.unlink('.debugfsinfo') - -@command('debuggetbundle', - [('H', 'head', [], _('id of head node'), _('ID')), - ('C', 'common', [], _('id of common node'), _('ID')), - ('t', 'type', 'bzip2', _('bundle compression type to use'), _('TYPE'))], - _('REPO FILE [-H|-C ID]...'), - norepo=True) -def debuggetbundle(ui, repopath, bundlepath, head=None, common=None, **opts): - """retrieves a bundle from a repo - - Every ID must be a full-length hex node id string. Saves the bundle to the - given file. - """ - repo = hg.peer(ui, opts, repopath) - if not repo.capable('getbundle'): - raise error.Abort("getbundle() not supported by target repository") - args = {} - if common: - args['common'] = [bin(s) for s in common] - if head: - args['heads'] = [bin(s) for s in head] - # TODO: get desired bundlecaps from command line. - args['bundlecaps'] = None - bundle = repo.getbundle('debug', **args) - - bundletype = opts.get('type', 'bzip2').lower() - btypes = {'none': 'HG10UN', - 'bzip2': 'HG10BZ', - 'gzip': 'HG10GZ', - 'bundle2': 'HG20'} - bundletype = btypes.get(bundletype) - if bundletype not in bundle2.bundletypes: - raise error.Abort(_('unknown bundle type specified with --type')) - bundle2.writebundle(ui, bundle, bundlepath, bundletype) - -@command('debugignore', [], '[FILE]') -def debugignore(ui, repo, *files, **opts): - """display the combined ignore pattern and information about ignored files - - With no argument display the combined ignore pattern. - - Given space separated file names, shows if the given file is ignored and - if so, show the ignore rule (file and line number) that matched it. - """ - ignore = repo.dirstate._ignore - if not files: - # Show all the patterns - includepat = getattr(ignore, 'includepat', None) - if includepat is not None: - ui.write("%s\n" % includepat) - else: - raise error.Abort(_("no ignore patterns found")) - else: - for f in files: - nf = util.normpath(f) - ignored = None - ignoredata = None - if nf != '.': - if ignore(nf): - ignored = nf - ignoredata = repo.dirstate._ignorefileandline(nf) - else: - for p in util.finddirs(nf): - if ignore(p): - ignored = p - ignoredata = repo.dirstate._ignorefileandline(p) - break - if ignored: - if ignored == nf: - ui.write(_("%s is ignored\n") % f) - else: - ui.write(_("%s is ignored because of " - "containing folder %s\n") - % (f, ignored)) - ignorefile, lineno, line = ignoredata - ui.write(_("(ignore rule in %s, line %d: '%s')\n") - % (ignorefile, lineno, line)) - else: - ui.write(_("%s is not ignored\n") % f) - -@command('debugindex', debugrevlogopts + - [('f', 'format', 0, _('revlog format'), _('FORMAT'))], - _('[-f FORMAT] -c|-m|FILE'), - optionalrepo=True) -def debugindex(ui, repo, file_=None, **opts): - """dump the contents of an index file""" - r = cmdutil.openrevlog(repo, 'debugindex', file_, opts) - format = opts.get('format', 0) - if format not in (0, 1): - raise error.Abort(_("unknown format %d") % format) - - generaldelta = r.version & revlog.REVLOGGENERALDELTA - if generaldelta: - basehdr = ' delta' - else: - basehdr = ' base' - - if ui.debugflag: - shortfn = hex - else: - shortfn = short - - # There might not be anything in r, so have a sane default - idlen = 12 - for i in r: - idlen = len(shortfn(r.node(i))) - break - - if format == 0: - ui.write((" rev offset length " + basehdr + " linkrev" - " %s %s p2\n") % ("nodeid".ljust(idlen), "p1".ljust(idlen))) - elif format == 1: - ui.write((" rev flag offset length" - " size " + basehdr + " link p1 p2" - " %s\n") % "nodeid".rjust(idlen)) - - for i in r: - node = r.node(i) - if generaldelta: - base = r.deltaparent(i) - else: - base = r.chainbase(i) - if format == 0: - try: - pp = r.parents(node) - except Exception: - pp = [nullid, nullid] - ui.write("% 6d % 9d % 7d % 6d % 7d %s %s %s\n" % ( - i, r.start(i), r.length(i), base, r.linkrev(i), - shortfn(node), shortfn(pp[0]), shortfn(pp[1]))) - elif format == 1: - pr = r.parentrevs(i) - ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % ( - i, r.flags(i), r.start(i), r.length(i), r.rawsize(i), - base, r.linkrev(i), pr[0], pr[1], shortfn(node))) - -@command('debugindexdot', debugrevlogopts, - _('-c|-m|FILE'), optionalrepo=True) -def debugindexdot(ui, repo, file_=None, **opts): - """dump an index DAG as a graphviz dot file""" - r = cmdutil.openrevlog(repo, 'debugindexdot', file_, opts) - ui.write(("digraph G {\n")) - for i in r: - node = r.node(i) - pp = r.parents(node) - ui.write("\t%d -> %d\n" % (r.rev(pp[0]), i)) - if pp[1] != nullid: - ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i)) - ui.write("}\n") - -@command('debugdeltachain', - debugrevlogopts + formatteropts, - _('-c|-m|FILE'), - optionalrepo=True) -def debugdeltachain(ui, repo, file_=None, **opts): - """dump information about delta chains in a revlog - - Output can be templatized. Available template keywords are: - - :``rev``: revision number - :``chainid``: delta chain identifier (numbered by unique base) - :``chainlen``: delta chain length to this revision - :``prevrev``: previous revision in delta chain - :``deltatype``: role of delta / how it was computed - :``compsize``: compressed size of revision - :``uncompsize``: uncompressed size of revision - :``chainsize``: total size of compressed revisions in chain - :``chainratio``: total chain size divided by uncompressed revision size - (new delta chains typically start at ratio 2.00) - :``lindist``: linear distance from base revision in delta chain to end - of this revision - :``extradist``: total size of revisions not part of this delta chain from - base of delta chain to end of this revision; a measurement - of how much extra data we need to read/seek across to read - the delta chain for this revision - :``extraratio``: extradist divided by chainsize; another representation of - how much unrelated data is needed to load this delta chain - """ - r = cmdutil.openrevlog(repo, 'debugdeltachain', file_, opts) - index = r.index - generaldelta = r.version & revlog.REVLOGGENERALDELTA - - def revinfo(rev): - e = index[rev] - compsize = e[1] - uncompsize = e[2] - chainsize = 0 - - if generaldelta: - if e[3] == e[5]: - deltatype = 'p1' - elif e[3] == e[6]: - deltatype = 'p2' - elif e[3] == rev - 1: - deltatype = 'prev' - elif e[3] == rev: - deltatype = 'base' - else: - deltatype = 'other' - else: - if e[3] == rev: - deltatype = 'base' - else: - deltatype = 'prev' - - chain = r._deltachain(rev)[0] - for iterrev in chain: - e = index[iterrev] - chainsize += e[1] - - return compsize, uncompsize, deltatype, chain, chainsize - - fm = ui.formatter('debugdeltachain', opts) - - fm.plain(' rev chain# chainlen prev delta ' - 'size rawsize chainsize ratio lindist extradist ' - 'extraratio\n') - - chainbases = {} - for rev in r: - comp, uncomp, deltatype, chain, chainsize = revinfo(rev) - chainbase = chain[0] - chainid = chainbases.setdefault(chainbase, len(chainbases) + 1) - basestart = r.start(chainbase) - revstart = r.start(rev) - lineardist = revstart + comp - basestart - extradist = lineardist - chainsize - try: - prevrev = chain[-2] - except IndexError: - prevrev = -1 - - chainratio = float(chainsize) / float(uncomp) - extraratio = float(extradist) / float(chainsize) - - fm.startitem() - fm.write('rev chainid chainlen prevrev deltatype compsize ' - 'uncompsize chainsize chainratio lindist extradist ' - 'extraratio', - '%7d %7d %8d %8d %7s %10d %10d %10d %9.5f %9d %9d %10.5f\n', - rev, chainid, len(chain), prevrev, deltatype, comp, - uncomp, chainsize, chainratio, lineardist, extradist, - extraratio, - rev=rev, chainid=chainid, chainlen=len(chain), - prevrev=prevrev, deltatype=deltatype, compsize=comp, - uncompsize=uncomp, chainsize=chainsize, - chainratio=chainratio, lindist=lineardist, - extradist=extradist, extraratio=extraratio) - - fm.end() - @command('debuginstall', [] + formatteropts, '', norepo=True) def debuginstall(ui, **opts): '''test Mercurial installation @@ -2698,9 +1898,9 @@ # Python fm.write('pythonexe', _("checking Python executable (%s)\n"), - sys.executable) + pycompat.sysexecutable) fm.write('pythonver', _("checking Python version (%s)\n"), - ("%s.%s.%s" % sys.version_info[:3])) + ("%d.%d.%d" % sys.version_info[:3])) fm.write('pythonlib', _("checking Python lib (%s)...\n"), os.path.dirname(os.__file__)) @@ -2750,6 +1950,22 @@ problems += 1 fm.condwrite(err, 'extensionserror', " %s\n", err) + compengines = util.compengines._engines.values() + fm.write('compengines', _('checking registered compression engines (%s)\n'), + fm.formatlist(sorted(e.name() for e in compengines), + name='compengine', fmt='%s', sep=', ')) + fm.write('compenginesavail', _('checking available compression engines ' + '(%s)\n'), + fm.formatlist(sorted(e.name() for e in compengines + if e.available()), + name='compengine', fmt='%s', sep=', ')) + wirecompengines = util.compengines.supportedwireengines(util.SERVERROLE) + fm.write('compenginesserver', _('checking available compression engines ' + 'for wire protocol (%s)\n'), + fm.formatlist([e.name() for e in wirecompengines + if e.wireprotosupport()], + name='compengine', fmt='%s', sep=', ')) + # templates p = templater.templatepaths() fm.write('templatedirs', 'checking templates (%s)...\n', ' '.join(p)) @@ -2780,7 +1996,7 @@ editor = ui.geteditor() editor = util.expandpath(editor) fm.write('editor', _("checking commit editor... (%s)\n"), editor) - cmdpath = util.findexe(shlex.split(editor)[0]) + cmdpath = util.findexe(pycompat.shlexsplit(editor)[0]) fm.condwrite(not cmdpath and editor == 'vi', 'vinotfound', _(" No commit editor set and can't find %s in PATH\n" " (specify a commit editor in your configuration" @@ -3067,7 +2283,7 @@ with repo.lock(): n = repair.deleteobsmarkers(repo.obsstore, indices) - ui.write(_('deleted %i obsolescense markers\n') % n) + ui.write(_('deleted %i obsolescence markers\n') % n) return @@ -3153,16 +2369,16 @@ def complete(path, acceptable): dirstate = repo.dirstate - spec = os.path.normpath(os.path.join(os.getcwd(), path)) - rootdir = repo.root + os.sep + spec = os.path.normpath(os.path.join(pycompat.getcwd(), path)) + rootdir = repo.root + pycompat.ossep if spec != repo.root and not spec.startswith(rootdir): return [], [] if os.path.isdir(spec): spec += '/' spec = spec[len(rootdir):] - fixpaths = os.sep != '/' + fixpaths = pycompat.ossep != '/' if fixpaths: - spec = spec.replace(os.sep, '/') + spec = spec.replace(pycompat.ossep, '/') speclen = len(spec) fullpaths = opts['full'] files, dirs = set(), set() @@ -3170,11 +2386,11 @@ for f, st in dirstate.iteritems(): if f.startswith(spec) and st[0] in acceptable: if fixpaths: - f = f.replace('/', os.sep) + f = f.replace('/', pycompat.ossep) if fullpaths: addfile(f) continue - s = f.find(os.sep, speclen) + s = f.find(pycompat.ossep, speclen) if s >= 0: adddir(f[:s]) else: @@ -3366,6 +2582,8 @@ datasize = [None, 0, 0] fullsize = [None, 0, 0] deltasize = [None, 0, 0] + chunktypecounts = {} + chunktypesizes = {} def addsize(size, l): if l[0] is None or size < l[0]: @@ -3403,6 +2621,20 @@ elif delta != nullrev: numother += 1 + # Obtain data on the raw chunks in the revlog. + chunk = r._chunkraw(rev, rev)[1] + if chunk: + chunktype = chunk[0] + else: + chunktype = 'empty' + + if chunktype not in chunktypecounts: + chunktypecounts[chunktype] = 0 + chunktypesizes[chunktype] = 0 + + chunktypecounts[chunktype] += 1 + chunktypesizes[chunktype] += size + # Adjust size min value for empty cases for size in (datasize, fullsize, deltasize): if size[0] is None: @@ -3454,6 +2686,24 @@ ui.write((' full : ') + fmt % pcfmt(fulltotal, totalsize)) ui.write((' deltas : ') + fmt % pcfmt(deltatotal, totalsize)) + def fmtchunktype(chunktype): + if chunktype == 'empty': + return ' %s : ' % chunktype + elif chunktype in string.ascii_letters: + return ' 0x%s (%s) : ' % (hex(chunktype), chunktype) + else: + return ' 0x%s : ' % hex(chunktype) + + ui.write('\n') + ui.write(('chunks : ') + fmt2 % numrevs) + for chunktype in sorted(chunktypecounts): + ui.write(fmtchunktype(chunktype)) + ui.write(fmt % pcfmt(chunktypecounts[chunktype], numrevs)) + ui.write(('chunks size : ') + fmt2 % totalsize) + for chunktype in sorted(chunktypecounts): + ui.write(fmtchunktype(chunktype)) + ui.write(fmt % pcfmt(chunktypesizes[chunktype], totalsize)) + ui.write('\n') fmt = dfmtstr(max(avgchainlen, compratio)) ui.write(('avg chain length : ') + fmt % avgchainlen) @@ -3755,7 +3005,7 @@ if not items: return f = lambda fn: fn - if ui.configbool('ui', 'slash') and os.sep != '/': + if ui.configbool('ui', 'slash') and pycompat.ossep != '/': f = lambda fn: util.normpath(fn) fmt = 'f %%-%ds %%-%ds %%s' % ( max([len(abs) for abs in items]), @@ -4120,8 +3370,7 @@ hg log -r "sort(all(), date)" - See :hg:`help revisions` and :hg:`help revsets` for more about - specifying revisions. + See :hg:`help revisions` for more about specifying revisions. Returns 0 on successful completion. ''' @@ -4655,15 +3904,15 @@ keep = opts.get('system') or [] if len(keep) == 0: - if sys.platform.startswith('win'): + if pycompat.sysplatform.startswith('win'): keep.append('windows') - elif sys.platform == 'OpenVMS': + elif pycompat.sysplatform == 'OpenVMS': keep.append('vms') - elif sys.platform == 'plan9': + elif pycompat.sysplatform == 'plan9': keep.append('plan9') else: keep.append('unix') - keep.append(sys.platform.lower()) + keep.append(pycompat.sysplatform.lower()) if ui.verbose: keep.append('verbose') @@ -5011,7 +4260,7 @@ lock = repo.lock() tr = repo.transaction('import') else: - dsguard = cmdutil.dirstateguard(repo, 'import') + dsguard = dirstateguard.dirstateguard(repo, 'import') parents = repo[None].parents() for patchurl in patches: if patchurl == '-': @@ -5330,8 +4579,8 @@ See :hg:`help dates` for a list of formats valid for -d/--date. - See :hg:`help revisions` and :hg:`help revsets` for more about - specifying and ordering revisions. + See :hg:`help revisions` for more about specifying and ordering + revisions. See :hg:`help templates` for more about pre-packaged styles and specifying custom templates. @@ -6574,11 +5823,8 @@ s = sshserver.sshserver(ui, repo) s.serve_forever() - if opts["cmdserver"]: - service = commandserver.createservice(ui, repo, opts) - else: - service = hgweb.createservice(ui, repo, opts) - return cmdutil.service(opts, initfn=service.init, runfn=service.run) + service = server.createservice(ui, repo, opts) + return server.runservice(opts, initfn=service.init, runfn=service.run) @command('^status|st', [('A', 'all', None, _('show status of all files')), @@ -6751,7 +5997,7 @@ # shows a working directory parent *changeset*: # i18n: column positioning for "hg summary" ui.write(_('parent: %d:%s ') % (p.rev(), str(p)), - label='log.changeset changeset.%s' % p.phasestr()) + label=cmdutil._changesetlabels(p)) ui.write(' '.join(p.tags()), label='log.tag') if p.bookmarks(): marks.extend(p.bookmarks()) @@ -6760,6 +6006,11 @@ ui.write(_(' (empty repository)')) else: ui.write(_(' (no revision checked out)')) + if p.troubled(): + ui.write(' (' + + ', '.join(ui.label(trouble, 'trouble.%s' % trouble) + for trouble in p.troubles()) + + ')') ui.write('\n') if p.description(): ui.status(' ' + p.description().splitlines()[0].strip() + '\n', @@ -7230,10 +6481,10 @@ changeset (see :hg:`help parents`). If the changeset is not a descendant or ancestor of the working - directory's parent, the update is aborted. With the -c/--check - option, the working directory is checked for uncommitted changes; if - none are found, the working directory is updated to the specified - changeset. + directory's parent and there are uncommitted changes, the update is + aborted. With the -c/--check option, the working directory is checked + for uncommitted changes; if none are found, the working directory is + updated to the specified changeset. .. container:: verbose diff -Nru mercurial-4.0.2~trusty1/mercurial/config.py mercurial-4.1~rc~trusty1/mercurial/config.py --- mercurial-4.0.2~trusty1/mercurial/config.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/config.py 2017-01-18 16:54:51.000000000 +0000 @@ -90,13 +90,13 @@ self._source.pop((section, item), None) def parse(self, src, data, sections=None, remap=None, include=None): - sectionre = util.re.compile(r'\[([^\[]+)\]') - itemre = util.re.compile(r'([^=\s][^=]*?)\s*=\s*(.*\S|)') - contre = util.re.compile(r'\s+(\S|\S.*\S)\s*$') - emptyre = util.re.compile(r'(;|#|\s*$)') - commentre = util.re.compile(r'(;|#)') - unsetre = util.re.compile(r'%unset\s+(\S+)') - includere = util.re.compile(r'%include\s+(\S|\S.*\S)\s*$') + sectionre = util.re.compile(br'\[([^\[]+)\]') + itemre = util.re.compile(br'([^=\s][^=]*?)\s*=\s*(.*\S|)') + contre = util.re.compile(br'\s+(\S|\S.*\S)\s*$') + emptyre = util.re.compile(br'(;|#|\s*$)') + commentre = util.re.compile(br'(;|#)') + unsetre = util.re.compile(br'%unset\s+(\S+)') + includere = util.re.compile(br'%include\s+(\S|\S.*\S)\s*$') section = "" item = None line = 0 diff -Nru mercurial-4.0.2~trusty1/mercurial/context.py mercurial-4.1~rc~trusty1/mercurial/context.py --- mercurial-4.0.2~trusty1/mercurial/context.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/context.py 2017-01-18 16:54:51.000000000 +0000 @@ -14,8 +14,11 @@ from .i18n import _ from .node import ( + addednodeid, bin, hex, + modifiednodeid, + newnodeid, nullid, nullrev, short, @@ -39,11 +42,6 @@ propertycache = util.propertycache -# Phony node value to stand-in for new files in some uses of -# manifests. Manifests support 21-byte hashes for nodes which are -# dirty in the working copy. -_newnode = '!' * 21 - nonascii = re.compile(r'[^\x21-\x7f]').search class basectx(object): @@ -142,7 +140,7 @@ removed.append(fn) elif flag1 != flag2: modified.append(fn) - elif node2 != _newnode: + elif node2 != newnodeid: # When comparing files between two commits, we save time by # not comparing the file contents when the nodeids differ. # Note that this means we incorrectly report a reverted change @@ -178,6 +176,8 @@ return hex(self.node()) def manifest(self): return self._manifest + def manifestctx(self): + return self._manifestctx def repo(self): return self._repo def phasestr(self): @@ -259,8 +259,10 @@ if path in self._manifestdelta: return (self._manifestdelta[path], self._manifestdelta.flags(path)) - node, flag = self._repo.manifest.find(self._changeset.manifest, path) - if not node: + mfl = self._repo.manifestlog + try: + node, flag = mfl[self._changeset.manifest].find(path) + except KeyError: raise error.ManifestLookupError(self._node, path, _('not found in manifest')) @@ -528,12 +530,15 @@ @propertycache def _manifest(self): - return self._repo.manifestlog[self._changeset.manifest].read() + return self._manifestctx.read() + + @propertycache + def _manifestctx(self): + return self._repo.manifestlog[self._changeset.manifest] @propertycache def _manifestdelta(self): - mfnode = self._changeset.manifest - return self._repo.manifestlog[mfnode].readdelta() + return self._manifestctx.readdelta() @propertycache def _parents(self): @@ -680,8 +685,7 @@ elif '_descendantrev' in self.__dict__: # this file context was created from a revision with a known # descendant, we can (lazily) correct for linkrev aliases - return self._adjustlinkrev(self._path, self._filelog, - self._filenode, self._descendantrev) + return self._adjustlinkrev(self._descendantrev) else: return self._filelog.linkrev(self._filerev) @@ -709,7 +713,10 @@ return False def __str__(self): - return "%s@%s" % (self.path(), self._changectx) + try: + return "%s@%s" % (self.path(), self._changectx) + except error.LookupError: + return "%s@???" % self.path() def __repr__(self): return "<%s %s>" % (type(self).__name__, str(self)) @@ -808,17 +815,13 @@ return True - def _adjustlinkrev(self, path, filelog, fnode, srcrev, inclusive=False): + def _adjustlinkrev(self, srcrev, inclusive=False): """return the first ancestor of introducing If the linkrev of the file revision does not point to an ancestor of srcrev, we'll walk down the ancestors until we find one introducing this file revision. - :repo: a localrepository object (used to access changelog and manifest) - :path: the file path - :fnode: the nodeid of the file revision - :filelog: the filelog of this path :srcrev: the changeset revision we search ancestors from :inclusive: if true, the src revision will also be checked """ @@ -826,8 +829,7 @@ cl = repo.unfiltered().changelog mfl = repo.manifestlog # fetch the linkrev - fr = filelog.rev(fnode) - lkr = filelog.linkrev(fr) + lkr = self.linkrev() # hack to reuse ancestor computation when searching for renames memberanc = getattr(self, '_ancestrycontext', None) iteranc = None @@ -844,6 +846,8 @@ if lkr not in memberanc: if iteranc is None: iteranc = cl.ancestors(revs, lkr, inclusive=inclusive) + fnode = self._filenode + path = self._path for a in iteranc: ac = cl.read(a) # get changeset data (we avoid object creation) if path in ac[3]: # checking the 'files' field. @@ -871,8 +875,7 @@ noctx = not ('_changeid' in attrs or '_changectx' in attrs) if noctx or self.rev() == lkr: return self.linkrev() - return self._adjustlinkrev(self._path, self._filelog, self._filenode, - self.rev(), inclusive=True) + return self._adjustlinkrev(self.rev(), inclusive=True) def _parentfilectx(self, path, fileid, filelog): """create parent filectx keeping ancestry info for _adjustlinkrev()""" @@ -1107,6 +1110,9 @@ return filectx(self._repo, self._path, fileid=fileid, filelog=self._filelog, changeid=changeid) + def rawdata(self): + return self._filelog.revision(self._filenode, raw=True) + def data(self): try: return self._filelog.read(self._filenode) @@ -1150,6 +1156,42 @@ return [filectx(self._repo, self._path, fileid=x, filelog=self._filelog) for x in c] +def _changesrange(fctx1, fctx2, linerange2, diffopts): + """Return `(diffinrange, linerange1)` where `diffinrange` is True + if diff from fctx2 to fctx1 has changes in linerange2 and + `linerange1` is the new line range for fctx1. + """ + blocks = mdiff.allblocks(fctx1.data(), fctx2.data(), diffopts) + filteredblocks, linerange1 = mdiff.blocksinrange(blocks, linerange2) + diffinrange = any(stype == '!' for _, stype in filteredblocks) + return diffinrange, linerange1 + +def blockancestors(fctx, fromline, toline): + """Yield ancestors of `fctx` with respect to the block of lines within + `fromline`-`toline` range. + """ + diffopts = patch.diffopts(fctx._repo.ui) + visit = {(fctx.linkrev(), fctx.filenode()): (fctx, (fromline, toline))} + while visit: + c, linerange2 = visit.pop(max(visit)) + pl = c.parents() + if not pl: + # The block originates from the initial revision. + yield c + continue + inrange = False + for p in pl: + inrangep, linerange1 = _changesrange(p, c, linerange2, diffopts) + inrange = inrange or inrangep + if linerange1[0] == linerange1[1]: + # Parent's linerange is empty, meaning that the block got + # introduced in this revision; no need to go futher in this + # branch. + continue + visit[p.linkrev(), p.filenode()] = p, linerange1 + if inrange: + yield c + class committablectx(basectx): """A committablectx object provides common functionality for a context that wants the ability to commit, e.g. workingctx or memctx.""" @@ -1231,23 +1273,13 @@ """ parents = self.parents() - man1 = parents[0].manifest() - man = man1.copy() - if len(parents) > 1: - man2 = self.p2().manifest() - def getman(f): - if f in man1: - return man1 - return man2 - else: - getman = lambda f: man1 + man = parents[0].manifest().copy() - copied = self._repo.dirstate.copies() ff = self._flagfunc - for i, l in (("a", self._status.added), ("m", self._status.modified)): + for i, l in ((addednodeid, self._status.added), + (modifiednodeid, self._status.modified)): for f in l: - orig = copied.get(f, f) - man[f] = getman(orig).get(orig, nullid) + i + man[f] = i try: man.setflag(f, ff(f)) except OSError: @@ -1582,7 +1614,7 @@ """ mf = self._repo['.']._manifestmatches(match, s) for f in s.modified + s.added: - mf[f] = _newnode + mf[f] = newnodeid mf.setflag(f, self.flags(f)) for f in s.removed: if f in mf: @@ -1982,3 +2014,101 @@ def write(self, data, flags): """wraps repo.wwrite""" self._data = data + +class metadataonlyctx(committablectx): + """Like memctx but it's reusing the manifest of different commit. + Intended to be used by lightweight operations that are creating + metadata-only changes. + + Revision information is supplied at initialization time. 'repo' is the + current localrepo, 'ctx' is original revision which manifest we're reuisng + 'parents' is a sequence of two parent revisions identifiers (pass None for + every missing parent), 'text' is the commit. + + user receives the committer name and defaults to current repository + username, date is the commit date in any format supported by + util.parsedate() and defaults to current date, extra is a dictionary of + metadata or is left empty. + """ + def __new__(cls, repo, originalctx, *args, **kwargs): + return super(metadataonlyctx, cls).__new__(cls, repo) + + def __init__(self, repo, originalctx, parents, text, user=None, date=None, + extra=None, editor=False): + super(metadataonlyctx, self).__init__(repo, text, user, date, extra) + self._rev = None + self._node = None + self._originalctx = originalctx + self._manifestnode = originalctx.manifestnode() + parents = [(p or nullid) for p in parents] + p1, p2 = self._parents = [changectx(self._repo, p) for p in parents] + + # sanity check to ensure that the reused manifest parents are + # manifests of our commit parents + mp1, mp2 = self.manifestctx().parents + if p1 != nullid and p1.manifestctx().node() != mp1: + raise RuntimeError('can\'t reuse the manifest: ' + 'its p1 doesn\'t match the new ctx p1') + if p2 != nullid and p2.manifestctx().node() != mp2: + raise RuntimeError('can\'t reuse the manifest: ' + 'its p2 doesn\'t match the new ctx p2') + + self._files = originalctx.files() + self.substate = {} + + if extra: + self._extra = extra.copy() + else: + self._extra = {} + + if self._extra.get('branch', '') == '': + self._extra['branch'] = 'default' + + if editor: + self._text = editor(self._repo, self, []) + self._repo.savecommitmessage(self._text) + + def manifestnode(self): + return self._manifestnode + + @propertycache + def _manifestctx(self): + return self._repo.manifestlog[self._manifestnode] + + def filectx(self, path, filelog=None): + return self._originalctx.filectx(path, filelog=filelog) + + def commit(self): + """commit context to the repo""" + return self._repo.commitctx(self) + + @property + def _manifest(self): + return self._originalctx.manifest() + + @propertycache + def _status(self): + """Calculate exact status from ``files`` specified in the ``origctx`` + and parents manifests. + """ + man1 = self.p1().manifest() + p2 = self._parents[1] + # "1 < len(self._parents)" can't be used for checking + # existence of the 2nd parent, because "metadataonlyctx._parents" is + # explicitly initialized by the list, of which length is 2. + if p2.node() != nullid: + man2 = p2.manifest() + managing = lambda f: f in man1 or f in man2 + else: + managing = lambda f: f in man1 + + modified, added, removed = [], [], [] + for f in self._files: + if not managing(f): + added.append(f) + elif self[f]: + modified.append(f) + else: + removed.append(f) + + return scmutil.status(modified, added, removed, [], [], [], []) diff -Nru mercurial-4.0.2~trusty1/mercurial/copies.py mercurial-4.1~rc~trusty1/mercurial/copies.py --- mercurial-4.0.2~trusty1/mercurial/copies.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/copies.py 2017-01-18 16:54:51.000000000 +0000 @@ -278,7 +278,7 @@ ac = repo.changelog.ancestors(revs, inclusive=True) ctx._ancestrycontext = ac def makectx(f, n): - if len(n) != 20: # in a working context? + if n in node.wdirnodes: # in a working context? if ctx.rev() is None: return ctx.filectx(f) return repo[None][f] @@ -310,8 +310,8 @@ Find moves and copies between context c1 and c2 that are relevant for merging. 'base' will be used as the merge base. - Returns four dicts: "copy", "movewithdir", "diverge", and - "renamedelete". + Returns five dicts: "copy", "movewithdir", "diverge", "renamedelete" and + "dirmove". "copy" is a mapping from destination name -> source name, where source is in c1 and destination is in c2 or vice-versa. @@ -326,20 +326,24 @@ "renamedelete" is a mapping of source name -> list of destination names for files deleted in c1 that were renamed in c2 or vice-versa. + + "dirmove" is a mapping of detected source dir -> destination dir renames. + This is needed for handling changes to new files previously grafted into + renamed directories. """ # avoid silly behavior for update from empty dir if not c1 or not c2 or c1 == c2: - return {}, {}, {}, {} + return {}, {}, {}, {}, {} # avoid silly behavior for parent -> working dir if c2.node() is None and c1.node() == repo.dirstate.p1(): - return repo.dirstate.copies(), {}, {}, {} + return repo.dirstate.copies(), {}, {}, {}, {} # Copy trace disabling is explicitly below the node == p1 logic above # because the logic above is required for a simple copy to be kept across a # rebase. if repo.ui.configbool('experimental', 'disablecopytrace'): - return {}, {}, {}, {} + return {}, {}, {}, {}, {} # In certain scenarios (e.g. graft, update or rebase), base can be # overridden We still need to know a real common ancestor in this case We @@ -365,7 +369,7 @@ limit = _findlimit(repo, c1.rev(), c2.rev()) if limit is None: # no common ancestor, no copies - return {}, {}, {}, {} + return {}, {}, {}, {}, {} repo.ui.debug(" searching for copies back to rev %d\n" % limit) m1 = c1.manifest() @@ -503,7 +507,7 @@ del divergeset if not fullcopy: - return copy, {}, diverge, renamedelete + return copy, {}, diverge, renamedelete, {} repo.ui.debug(" checking for directory renames\n") @@ -541,7 +545,7 @@ del d1, d2, invalid if not dirmove: - return copy, {}, diverge, renamedelete + return copy, {}, diverge, renamedelete, {} for d in dirmove: repo.ui.debug(" discovered dir src: '%s' -> dst: '%s'\n" % @@ -561,7 +565,7 @@ "dst: '%s'\n") % (f, df)) break - return copy, movewithdir, diverge, renamedelete + return copy, movewithdir, diverge, renamedelete, dirmove def _related(f1, f2, limit): """return True if f1 and f2 filectx have a common ancestor diff -Nru mercurial-4.0.2~trusty1/mercurial/crecord.py mercurial-4.1~rc~trusty1/mercurial/crecord.py --- mercurial-4.0.2~trusty1/mercurial/crecord.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/crecord.py 2017-01-18 16:54:51.000000000 +0000 @@ -14,14 +14,13 @@ import os import re import signal -import struct -import sys from .i18n import _ from . import ( encoding, error, patch as patchmod, + scmutil, util, ) stringio = util.stringio @@ -52,11 +51,7 @@ try: import curses - import fcntl - import termios curses.error - fcntl.ioctl - termios.TIOCGWINSZ except ImportError: # I have no idea if wcurses works with crecord... try: @@ -75,8 +70,6 @@ """ return curses and ui.interface("chunkselector") == "curses" -_origstdout = sys.__stdout__ # used by gethw() - class patchnode(object): """abstract class for patch graph nodes (i.e. patchroot, header, hunk, hunkline) @@ -440,7 +433,7 @@ def __repr__(self): return '' % (self.filename(), self.fromline) -def filterpatch(ui, chunks, chunkselector): +def filterpatch(ui, chunks, chunkselector, operation=None): """interactively filter patch chunks into applied-only chunks""" chunks = list(chunks) # convert chunks list into structure suitable for displaying/modifying @@ -453,7 +446,7 @@ uiheaders = [uiheader(h) for h in headers] # let user choose headers/hunks/lines, and mark their applied flags # accordingly - ret = chunkselector(ui, uiheaders) + ret = chunkselector(ui, uiheaders, operation=operation) appliedhunklist = [] for hdr in uiheaders: if (hdr.applied and @@ -473,25 +466,13 @@ return (appliedhunklist, ret) -def gethw(): - """ - magically get the current height and width of the window (without initscr) - - this is a rip-off of a rip-off - taken from the bpython code. it is - useful / necessary because otherwise curses.initscr() must be called, - which can leave the terminal in a nasty state after exiting. - """ - h, w = struct.unpack( - "hhhh", fcntl.ioctl(_origstdout, termios.TIOCGWINSZ, "\000"*8))[0:2] - return h, w - -def chunkselector(ui, headerlist): +def chunkselector(ui, headerlist, operation=None): """ curses interface to get selection of chunks, and mark the applied flags of the chosen chunks. """ ui.write(_('starting interactive selection\n')) - chunkselector = curseschunkselector(headerlist, ui) + chunkselector = curseschunkselector(headerlist, ui, operation) f = signal.getsignal(signal.SIGTSTP) curses.wrapper(chunkselector.main) if chunkselector.initerr is not None: @@ -505,12 +486,12 @@ return f(testfn, *args, **kwargs) return u -def testchunkselector(testfn, ui, headerlist): +def testchunkselector(testfn, ui, headerlist, operation=None): """ test interface to get selection of chunks, and mark the applied flags of the chosen chunks. """ - chunkselector = curseschunkselector(headerlist, ui) + chunkselector = curseschunkselector(headerlist, ui, operation) if testfn and os.path.exists(testfn): testf = open(testfn) testcommands = map(lambda x: x.rstrip('\n'), testf.readlines()) @@ -520,8 +501,14 @@ break return chunkselector.opts +_headermessages = { # {operation: text} + 'revert': _('Select hunks to revert'), + 'discard': _('Select hunks to discard'), + None: _('Select hunks to record'), +} + class curseschunkselector(object): - def __init__(self, headerlist, ui): + def __init__(self, headerlist, ui, operation=None): # put the headers into a patch object self.headerlist = patch(headerlist) @@ -560,7 +547,7 @@ # keeps track of the number of lines in the pad self.numpadlines = None - self.numstatuslines = 2 + self.numstatuslines = 1 # keep a running count of the number of lines printed to the pad # (used for determining when the selected item begins/ends) @@ -575,6 +562,11 @@ # if the last 'toggle all' command caused all changes to be applied self.waslasttoggleallapplied = True + # affects some ui text + if operation not in _headermessages: + raise RuntimeError('unexpected operation: %s' % operation) + self.operation = operation + def uparrowevent(self): """ try to select the previous item to the current item that has the @@ -962,6 +954,45 @@ self.linesprintedtopadsofar += linesprinted return t + def _getstatuslinesegments(self): + """-> [str]. return segments""" + selected = self.currentselecteditem.applied + segments = [ + _headermessages[self.operation], + '-', + _('[x]=selected **=collapsed'), + _('c: confirm'), + _('q: abort'), + _('arrow keys: move/expand/collapse'), + _('space: deselect') if selected else _('space: select'), + _('?: help'), + ] + return segments + + def _getstatuslines(self): + """() -> [str]. return short help used in the top status window""" + if self.errorstr is not None: + lines = [self.errorstr, _('Press any key to continue')] + else: + # wrap segments to lines + segments = self._getstatuslinesegments() + width = self.xscreensize + lines = [] + lastwidth = width + for s in segments: + w = encoding.colwidth(s) + sep = ' ' * (1 + (s and s[0] not in '-[')) + if lastwidth + w + len(sep) >= width: + lines.append(s) + lastwidth = w + else: + lines[-1] += sep + s + lastwidth += w + len(sep) + if len(lines) != self.numstatuslines: + self.numstatuslines = len(lines) + self.statuswin.resize(self.numstatuslines, self.xscreensize) + return [util.ellipsis(l, self.xscreensize - 1) for l in lines] + def updatescreen(self): self.statuswin.erase() self.chunkpad.erase() @@ -970,25 +1001,13 @@ # print out the status lines at the top try: - if self.errorstr is not None: - printstring(self.statuswin, self.errorstr, pairname='legend') - printstring(self.statuswin, 'Press any key to continue', - pairname='legend') - self.statuswin.refresh() - return - line1 = ("SELECT CHUNKS: (j/k/up/dn/pgup/pgdn) move cursor; " - "(space/A) toggle hunk/all; (e)dit hunk;") - line2 = (" (f)old/unfold; (c)onfirm applied; (q)uit; (?) help " - "| [X]=hunk applied **=folded, toggle [a]mend mode") - - printstring(self.statuswin, - util.ellipsis(line1, self.xscreensize - 1), - pairname="legend") - printstring(self.statuswin, - util.ellipsis(line2, self.xscreensize - 1), - pairname="legend") + for line in self._getstatuslines(): + printstring(self.statuswin, line, pairname="legend") + self.statuswin.refresh() except curses.error: pass + if self.errorstr is not None: + return # print out the patch in the remaining part of the window try: @@ -996,14 +1015,11 @@ self.updatescroll() self.chunkpad.refresh(self.firstlineofpadtoprint, 0, self.numstatuslines, 0, - self.yscreensize + 1 - self.numstatuslines, + self.yscreensize - self.numstatuslines, self.xscreensize) except curses.error: pass - # refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) - self.statuswin.refresh() - def getstatusprefixstring(self, item): """ create a string to prefix a line with which indicates whether 'item' @@ -1259,7 +1275,7 @@ "handle window resizing" try: curses.endwin() - self.yscreensize, self.xscreensize = gethw() + self.xscreensize, self.yscreensize = scmutil.termsize(self.ui) self.statuswin.resize(self.numstatuslines, self.xscreensize) self.numpadlines = self.getnumlinesdisplayed(ignorefolding=True) + 1 self.chunkpad = curses.newpad(self.numpadlines, self.xscreensize) @@ -1321,7 +1337,8 @@ def helpwindow(self): "print a help window to the screen. exit after any keypress." - helptext = """ [press any key to return to the patch-display] + helptext = _( + """ [press any key to return to the patch-display] crecord allows you to interactively choose among the changes you have made, and confirm only those changes you select for further processing by the command @@ -1345,7 +1362,7 @@ c : confirm selected changes r : review/edit and confirm selected changes q : quit without confirming (no changes will be made) - ? : help (what you're currently reading)""" + ? : help (what you're currently reading)""") helpwin = curses.newwin(self.yscreensize, 0, 0, 0) helplines = helptext.split("\n") @@ -1384,7 +1401,7 @@ def reviewcommit(self): """ask for 'y' to be pressed to confirm selected. return True if confirmed.""" - confirmtext = ( + confirmtext = _( """if you answer yes to the following, the your currently chosen patch chunks will be loaded into an editor. you may modify the patch from the editor, and save the changes if you wish to change the patch. otherwise, you can just @@ -1416,19 +1433,19 @@ except ValueError: ver = 1 if ver < 2.19: - msg = ("The amend option is unavailable with hg versions < 2.2\n\n" - "Press any key to continue.") + msg = _("The amend option is unavailable with hg versions < 2.2\n\n" + "Press any key to continue.") elif opts.get('amend') is None: opts['amend'] = True - msg = ("Amend option is turned on -- commiting the currently " - "selected changes will not create a new changeset, but " - "instead update the most recently committed changeset.\n\n" - "Press any key to continue.") + msg = _("Amend option is turned on -- committing the currently " + "selected changes will not create a new changeset, but " + "instead update the most recently committed changeset.\n\n" + "Press any key to continue.") elif opts.get('amend') is True: opts['amend'] = None - msg = ("Amend option is turned off -- commiting the currently " - "selected changes will create a new changeset.\n\n" - "Press any key to continue.") + msg = _("Amend option is turned off -- committing the currently " + "selected changes will create a new changeset.\n\n" + "Press any key to continue.") if not test: self.confirmationwindow(msg) @@ -1571,6 +1588,8 @@ return True elif keypressed in [' '] or (test and keypressed in ["TOGGLE"]): self.toggleapply() + if self.ui.configbool('experimental', 'spacemovesdown'): + self.downarrowevent() elif keypressed in ['A']: self.toggleall() elif keypressed in ['e']: @@ -1629,7 +1648,7 @@ except curses.error: self.initerr = _('this diff is too large to be displayed') return - # initialize selecteitemendline (initial start-line is 0) + # initialize selecteditemendline (initial start-line is 0) self.selecteditemendline = self.getnumlinesdisplayed( self.currentselecteditem, recursechildren=False) diff -Nru mercurial-4.0.2~trusty1/mercurial/debugcommands.py mercurial-4.1~rc~trusty1/mercurial/debugcommands.py --- mercurial-4.0.2~trusty1/mercurial/debugcommands.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/debugcommands.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,877 @@ +# debugcommands.py - command processing for debug* commands +# +# Copyright 2005-2016 Matt Mackall +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +import operator +import os +import random + +from .i18n import _ +from .node import ( + bin, + hex, + nullid, + short, +) +from . import ( + bundle2, + changegroup, + cmdutil, + commands, + context, + dagparser, + dagutil, + error, + exchange, + extensions, + fileset, + hg, + localrepo, + lock as lockmod, + pycompat, + repair, + revlog, + scmutil, + setdiscovery, + simplemerge, + streamclone, + treediscovery, + util, +) + +release = lockmod.release + +# We reuse the command table from commands because it is easier than +# teaching dispatch about multiple tables. +command = cmdutil.command(commands.table) + +@command('debugancestor', [], _('[INDEX] REV1 REV2'), optionalrepo=True) +def debugancestor(ui, repo, *args): + """find the ancestor revision of two revisions in a given index""" + if len(args) == 3: + index, rev1, rev2 = args + r = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), index) + lookup = r.lookup + elif len(args) == 2: + if not repo: + raise error.Abort(_('there is no Mercurial repository here ' + '(.hg not found)')) + rev1, rev2 = args + r = repo.changelog + lookup = repo.lookup + else: + raise error.Abort(_('either two or three arguments required')) + a = r.ancestor(lookup(rev1), lookup(rev2)) + ui.write('%d:%s\n' % (r.rev(a), hex(a))) + +@command('debugapplystreamclonebundle', [], 'FILE') +def debugapplystreamclonebundle(ui, repo, fname): + """apply a stream clone bundle file""" + f = hg.openpath(ui, fname) + gen = exchange.readbundle(ui, f, fname) + gen.apply(repo) + +@command('debugbuilddag', + [('m', 'mergeable-file', None, _('add single file mergeable changes')), + ('o', 'overwritten-file', None, _('add single file all revs overwrite')), + ('n', 'new-file', None, _('add new file at each rev'))], + _('[OPTION]... [TEXT]')) +def debugbuilddag(ui, repo, text=None, + mergeable_file=False, + overwritten_file=False, + new_file=False): + """builds a repo with a given DAG from scratch in the current empty repo + + The description of the DAG is read from stdin if not given on the + command line. + + Elements: + + - "+n" is a linear run of n nodes based on the current default parent + - "." is a single node based on the current default parent + - "$" resets the default parent to null (implied at the start); + otherwise the default parent is always the last node created + - " 0: + raise error.Abort(_('repository is not empty')) + + # determine number of revs in DAG + total = 0 + for type, data in dagparser.parsedag(text): + if type == 'n': + total += 1 + + if mergeable_file: + linesperrev = 2 + # make a file with k lines per rev + initialmergedlines = [str(i) for i in xrange(0, total * linesperrev)] + initialmergedlines.append("") + + tags = [] + + wlock = lock = tr = None + try: + wlock = repo.wlock() + lock = repo.lock() + tr = repo.transaction("builddag") + + at = -1 + atbranch = 'default' + nodeids = [] + id = 0 + ui.progress(_('building'), id, unit=_('revisions'), total=total) + for type, data in dagparser.parsedag(text): + if type == 'n': + ui.note(('node %s\n' % str(data))) + id, ps = data + + files = [] + fctxs = {} + + p2 = None + if mergeable_file: + fn = "mf" + p1 = repo[ps[0]] + if len(ps) > 1: + p2 = repo[ps[1]] + pa = p1.ancestor(p2) + base, local, other = [x[fn].data() for x in (pa, p1, + p2)] + m3 = simplemerge.Merge3Text(base, local, other) + ml = [l.strip() for l in m3.merge_lines()] + ml.append("") + elif at > 0: + ml = p1[fn].data().split("\n") + else: + ml = initialmergedlines + ml[id * linesperrev] += " r%i" % id + mergedtext = "\n".join(ml) + files.append(fn) + fctxs[fn] = context.memfilectx(repo, fn, mergedtext) + + if overwritten_file: + fn = "of" + files.append(fn) + fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) + + if new_file: + fn = "nf%i" % id + files.append(fn) + fctxs[fn] = context.memfilectx(repo, fn, "r%i\n" % id) + if len(ps) > 1: + if not p2: + p2 = repo[ps[1]] + for fn in p2: + if fn.startswith("nf"): + files.append(fn) + fctxs[fn] = p2[fn] + + def fctxfn(repo, cx, path): + return fctxs.get(path) + + if len(ps) == 0 or ps[0] < 0: + pars = [None, None] + elif len(ps) == 1: + pars = [nodeids[ps[0]], None] + else: + pars = [nodeids[p] for p in ps] + cx = context.memctx(repo, pars, "r%i" % id, files, fctxfn, + date=(id, 0), + user="debugbuilddag", + extra={'branch': atbranch}) + nodeid = repo.commitctx(cx) + nodeids.append(nodeid) + at = id + elif type == 'l': + id, name = data + ui.note(('tag %s\n' % name)) + tags.append("%s %s\n" % (hex(repo.changelog.node(id)), name)) + elif type == 'a': + ui.note(('branch %s\n' % data)) + atbranch = data + ui.progress(_('building'), id, unit=_('revisions'), total=total) + tr.close() + + if tags: + repo.vfs.write("localtags", "".join(tags)) + finally: + ui.progress(_('building'), None) + release(tr, lock, wlock) + +def _debugchangegroup(ui, gen, all=None, indent=0, **opts): + indent_string = ' ' * indent + if all: + ui.write(("%sformat: id, p1, p2, cset, delta base, len(delta)\n") + % indent_string) + + def showchunks(named): + ui.write("\n%s%s\n" % (indent_string, named)) + chain = None + for chunkdata in iter(lambda: gen.deltachunk(chain), {}): + node = chunkdata['node'] + p1 = chunkdata['p1'] + p2 = chunkdata['p2'] + cs = chunkdata['cs'] + deltabase = chunkdata['deltabase'] + delta = chunkdata['delta'] + ui.write("%s%s %s %s %s %s %s\n" % + (indent_string, hex(node), hex(p1), hex(p2), + hex(cs), hex(deltabase), len(delta))) + chain = node + + chunkdata = gen.changelogheader() + showchunks("changelog") + chunkdata = gen.manifestheader() + showchunks("manifest") + for chunkdata in iter(gen.filelogheader, {}): + fname = chunkdata['filename'] + showchunks(fname) + else: + if isinstance(gen, bundle2.unbundle20): + raise error.Abort(_('use debugbundle2 for this file')) + chunkdata = gen.changelogheader() + chain = None + for chunkdata in iter(lambda: gen.deltachunk(chain), {}): + node = chunkdata['node'] + ui.write("%s%s\n" % (indent_string, hex(node))) + chain = node + +def _debugbundle2(ui, gen, all=None, **opts): + """lists the contents of a bundle2""" + if not isinstance(gen, bundle2.unbundle20): + raise error.Abort(_('not a bundle2 file')) + ui.write(('Stream params: %s\n' % repr(gen.params))) + for part in gen.iterparts(): + ui.write('%s -- %r\n' % (part.type, repr(part.params))) + if part.type == 'changegroup': + version = part.params.get('version', '01') + cg = changegroup.getunbundler(version, part, 'UN') + _debugchangegroup(ui, cg, all=all, indent=4, **opts) + +@command('debugbundle', + [('a', 'all', None, _('show all details')), + ('', 'spec', None, _('print the bundlespec of the bundle'))], + _('FILE'), + norepo=True) +def debugbundle(ui, bundlepath, all=None, spec=None, **opts): + """lists the contents of a bundle""" + with hg.openpath(ui, bundlepath) as f: + if spec: + spec = exchange.getbundlespec(ui, f) + ui.write('%s\n' % spec) + return + + gen = exchange.readbundle(ui, f, bundlepath) + if isinstance(gen, bundle2.unbundle20): + return _debugbundle2(ui, gen, all=all, **opts) + _debugchangegroup(ui, gen, all=all, **opts) + +@command('debugcheckstate', [], '') +def debugcheckstate(ui, repo): + """validate the correctness of the current dirstate""" + parent1, parent2 = repo.dirstate.parents() + m1 = repo[parent1].manifest() + m2 = repo[parent2].manifest() + errors = 0 + for f in repo.dirstate: + state = repo.dirstate[f] + if state in "nr" and f not in m1: + ui.warn(_("%s in state %s, but not in manifest1\n") % (f, state)) + errors += 1 + if state in "a" and f in m1: + ui.warn(_("%s in state %s, but also in manifest1\n") % (f, state)) + errors += 1 + if state in "m" and f not in m1 and f not in m2: + ui.warn(_("%s in state %s, but not in either manifest\n") % + (f, state)) + errors += 1 + for f in m1: + state = repo.dirstate[f] + if state not in "nrm": + ui.warn(_("%s in manifest1, but listed as state %s") % (f, state)) + errors += 1 + if errors: + error = _(".hg/dirstate inconsistent with current parent's manifest") + raise error.Abort(error) + +@command('debugcommands', [], _('[COMMAND]'), norepo=True) +def debugcommands(ui, cmd='', *args): + """list all available commands and options""" + for cmd, vals in sorted(commands.table.iteritems()): + cmd = cmd.split('|')[0].strip('^') + opts = ', '.join([i[1] for i in vals[1]]) + ui.write('%s: %s\n' % (cmd, opts)) + +@command('debugcomplete', + [('o', 'options', None, _('show the command options'))], + _('[-o] CMD'), + norepo=True) +def debugcomplete(ui, cmd='', **opts): + """returns the completion list associated with the given command""" + + if opts.get('options'): + options = [] + otables = [commands.globalopts] + if cmd: + aliases, entry = cmdutil.findcmd(cmd, commands.table, False) + otables.append(entry[1]) + for t in otables: + for o in t: + if "(DEPRECATED)" in o[3]: + continue + if o[0]: + options.append('-%s' % o[0]) + options.append('--%s' % o[1]) + ui.write("%s\n" % "\n".join(options)) + return + + cmdlist, unused_allcmds = cmdutil.findpossible(cmd, commands.table) + if ui.verbose: + cmdlist = [' '.join(c[0]) for c in cmdlist.values()] + ui.write("%s\n" % "\n".join(sorted(cmdlist))) + +@command('debugcreatestreamclonebundle', [], 'FILE') +def debugcreatestreamclonebundle(ui, repo, fname): + """create a stream clone bundle file + + Stream bundles are special bundles that are essentially archives of + revlog files. They are commonly used for cloning very quickly. + """ + requirements, gen = streamclone.generatebundlev1(repo) + changegroup.writechunks(ui, gen, fname) + + ui.write(_('bundle requirements: %s\n') % ', '.join(sorted(requirements))) + +@command('debugdag', + [('t', 'tags', None, _('use tags as labels')), + ('b', 'branches', None, _('annotate with branch names')), + ('', 'dots', None, _('use dots for runs')), + ('s', 'spaces', None, _('separate elements by spaces'))], + _('[OPTION]... [FILE [REV]...]'), + optionalrepo=True) +def debugdag(ui, repo, file_=None, *revs, **opts): + """format the changelog or an index DAG as a concise textual description + + If you pass a revlog index, the revlog's DAG is emitted. If you list + revision numbers, they get labeled in the output as rN. + + Otherwise, the changelog DAG of the current repo is emitted. + """ + spaces = opts.get('spaces') + dots = opts.get('dots') + if file_: + rlog = revlog.revlog(scmutil.opener(pycompat.getcwd(), audit=False), + file_) + revs = set((int(r) for r in revs)) + def events(): + for r in rlog: + yield 'n', (r, list(p for p in rlog.parentrevs(r) + if p != -1)) + if r in revs: + yield 'l', (r, "r%i" % r) + elif repo: + cl = repo.changelog + tags = opts.get('tags') + branches = opts.get('branches') + if tags: + labels = {} + for l, n in repo.tags().items(): + labels.setdefault(cl.rev(n), []).append(l) + def events(): + b = "default" + for r in cl: + if branches: + newb = cl.read(cl.node(r))[5]['branch'] + if newb != b: + yield 'a', newb + b = newb + yield 'n', (r, list(p for p in cl.parentrevs(r) + if p != -1)) + if tags: + ls = labels.get(r) + if ls: + for l in ls: + yield 'l', (r, l) + else: + raise error.Abort(_('need repo for changelog dag')) + + for line in dagparser.dagtextlines(events(), + addspaces=spaces, + wraplabels=True, + wrapannotations=True, + wrapnonlinear=dots, + usedots=dots, + maxlinewidth=70): + ui.write(line) + ui.write("\n") + +@command('debugdata', commands.debugrevlogopts, _('-c|-m|FILE REV')) +def debugdata(ui, repo, file_, rev=None, **opts): + """dump the contents of a data file revision""" + if opts.get('changelog') or opts.get('manifest') or opts.get('dir'): + if rev is not None: + raise error.CommandError('debugdata', _('invalid arguments')) + file_, rev = None, file_ + elif rev is None: + raise error.CommandError('debugdata', _('invalid arguments')) + r = cmdutil.openrevlog(repo, 'debugdata', file_, opts) + try: + ui.write(r.revision(r.lookup(rev), raw=True)) + except KeyError: + raise error.Abort(_('invalid revision identifier %s') % rev) + +@command('debugdate', + [('e', 'extended', None, _('try extended date formats'))], + _('[-e] DATE [RANGE]'), + norepo=True, optionalrepo=True) +def debugdate(ui, date, range=None, **opts): + """parse and display a date""" + if opts["extended"]: + d = util.parsedate(date, util.extendeddateformats) + else: + d = util.parsedate(date) + ui.write(("internal: %s %s\n") % d) + ui.write(("standard: %s\n") % util.datestr(d)) + if range: + m = util.matchdate(range) + ui.write(("match: %s\n") % m(d[0])) + +@command('debugdeltachain', + commands.debugrevlogopts + commands.formatteropts, + _('-c|-m|FILE'), + optionalrepo=True) +def debugdeltachain(ui, repo, file_=None, **opts): + """dump information about delta chains in a revlog + + Output can be templatized. Available template keywords are: + + :``rev``: revision number + :``chainid``: delta chain identifier (numbered by unique base) + :``chainlen``: delta chain length to this revision + :``prevrev``: previous revision in delta chain + :``deltatype``: role of delta / how it was computed + :``compsize``: compressed size of revision + :``uncompsize``: uncompressed size of revision + :``chainsize``: total size of compressed revisions in chain + :``chainratio``: total chain size divided by uncompressed revision size + (new delta chains typically start at ratio 2.00) + :``lindist``: linear distance from base revision in delta chain to end + of this revision + :``extradist``: total size of revisions not part of this delta chain from + base of delta chain to end of this revision; a measurement + of how much extra data we need to read/seek across to read + the delta chain for this revision + :``extraratio``: extradist divided by chainsize; another representation of + how much unrelated data is needed to load this delta chain + """ + r = cmdutil.openrevlog(repo, 'debugdeltachain', file_, opts) + index = r.index + generaldelta = r.version & revlog.REVLOGGENERALDELTA + + def revinfo(rev): + e = index[rev] + compsize = e[1] + uncompsize = e[2] + chainsize = 0 + + if generaldelta: + if e[3] == e[5]: + deltatype = 'p1' + elif e[3] == e[6]: + deltatype = 'p2' + elif e[3] == rev - 1: + deltatype = 'prev' + elif e[3] == rev: + deltatype = 'base' + else: + deltatype = 'other' + else: + if e[3] == rev: + deltatype = 'base' + else: + deltatype = 'prev' + + chain = r._deltachain(rev)[0] + for iterrev in chain: + e = index[iterrev] + chainsize += e[1] + + return compsize, uncompsize, deltatype, chain, chainsize + + fm = ui.formatter('debugdeltachain', opts) + + fm.plain(' rev chain# chainlen prev delta ' + 'size rawsize chainsize ratio lindist extradist ' + 'extraratio\n') + + chainbases = {} + for rev in r: + comp, uncomp, deltatype, chain, chainsize = revinfo(rev) + chainbase = chain[0] + chainid = chainbases.setdefault(chainbase, len(chainbases) + 1) + basestart = r.start(chainbase) + revstart = r.start(rev) + lineardist = revstart + comp - basestart + extradist = lineardist - chainsize + try: + prevrev = chain[-2] + except IndexError: + prevrev = -1 + + chainratio = float(chainsize) / float(uncomp) + extraratio = float(extradist) / float(chainsize) + + fm.startitem() + fm.write('rev chainid chainlen prevrev deltatype compsize ' + 'uncompsize chainsize chainratio lindist extradist ' + 'extraratio', + '%7d %7d %8d %8d %7s %10d %10d %10d %9.5f %9d %9d %10.5f\n', + rev, chainid, len(chain), prevrev, deltatype, comp, + uncomp, chainsize, chainratio, lineardist, extradist, + extraratio, + rev=rev, chainid=chainid, chainlen=len(chain), + prevrev=prevrev, deltatype=deltatype, compsize=comp, + uncompsize=uncomp, chainsize=chainsize, + chainratio=chainratio, lindist=lineardist, + extradist=extradist, extraratio=extraratio) + + fm.end() + +@command('debugdiscovery', + [('', 'old', None, _('use old-style discovery')), + ('', 'nonheads', None, + _('use old-style discovery with non-heads included')), + ] + commands.remoteopts, + _('[-l REV] [-r REV] [-b BRANCH]... [OTHER]')) +def debugdiscovery(ui, repo, remoteurl="default", **opts): + """runs the changeset discovery protocol in isolation""" + remoteurl, branches = hg.parseurl(ui.expandpath(remoteurl), + opts.get('branch')) + remote = hg.peer(repo, opts, remoteurl) + ui.status(_('comparing with %s\n') % util.hidepassword(remoteurl)) + + # make sure tests are repeatable + random.seed(12323) + + def doit(localheads, remoteheads, remote=remote): + if opts.get('old'): + if localheads: + raise error.Abort('cannot use localheads with old style ' + 'discovery') + if not util.safehasattr(remote, 'branches'): + # enable in-client legacy support + remote = localrepo.locallegacypeer(remote.local()) + common, _in, hds = treediscovery.findcommonincoming(repo, remote, + force=True) + common = set(common) + if not opts.get('nonheads'): + ui.write(("unpruned common: %s\n") % + " ".join(sorted(short(n) for n in common))) + dag = dagutil.revlogdag(repo.changelog) + all = dag.ancestorset(dag.internalizeall(common)) + common = dag.externalizeall(dag.headsetofconnecteds(all)) + else: + common, any, hds = setdiscovery.findcommonheads(ui, repo, remote) + common = set(common) + rheads = set(hds) + lheads = set(repo.heads()) + ui.write(("common heads: %s\n") % + " ".join(sorted(short(n) for n in common))) + if lheads <= common: + ui.write(("local is subset\n")) + elif rheads <= common: + ui.write(("remote is subset\n")) + + serverlogs = opts.get('serverlog') + if serverlogs: + for filename in serverlogs: + with open(filename, 'r') as logfile: + line = logfile.readline() + while line: + parts = line.strip().split(';') + op = parts[1] + if op == 'cg': + pass + elif op == 'cgss': + doit(parts[2].split(' '), parts[3].split(' ')) + elif op == 'unb': + doit(parts[3].split(' '), parts[2].split(' ')) + line = logfile.readline() + else: + remoterevs, _checkout = hg.addbranchrevs(repo, remote, branches, + opts.get('remote_head')) + localrevs = opts.get('local_head') + doit(localrevs, remoterevs) + +@command('debugextensions', commands.formatteropts, [], norepo=True) +def debugextensions(ui, **opts): + '''show information about active extensions''' + exts = extensions.extensions(ui) + hgver = util.version() + fm = ui.formatter('debugextensions', opts) + for extname, extmod in sorted(exts, key=operator.itemgetter(0)): + isinternal = extensions.ismoduleinternal(extmod) + extsource = extmod.__file__ + if isinternal: + exttestedwith = [] # never expose magic string to users + else: + exttestedwith = getattr(extmod, 'testedwith', '').split() + extbuglink = getattr(extmod, 'buglink', None) + + fm.startitem() + + if ui.quiet or ui.verbose: + fm.write('name', '%s\n', extname) + else: + fm.write('name', '%s', extname) + if isinternal or hgver in exttestedwith: + fm.plain('\n') + elif not exttestedwith: + fm.plain(_(' (untested!)\n')) + else: + lasttestedversion = exttestedwith[-1] + fm.plain(' (%s!)\n' % lasttestedversion) + + fm.condwrite(ui.verbose and extsource, 'source', + _(' location: %s\n'), extsource or "") + + if ui.verbose: + fm.plain(_(' bundled: %s\n') % ['no', 'yes'][isinternal]) + fm.data(bundled=isinternal) + + fm.condwrite(ui.verbose and exttestedwith, 'testedwith', + _(' tested with: %s\n'), + fm.formatlist(exttestedwith, name='ver')) + + fm.condwrite(ui.verbose and extbuglink, 'buglink', + _(' bug reporting: %s\n'), extbuglink or "") + + fm.end() + +@command('debugfileset', + [('r', 'rev', '', _('apply the filespec on this revision'), _('REV'))], + _('[-r REV] FILESPEC')) +def debugfileset(ui, repo, expr, **opts): + '''parse and apply a fileset specification''' + ctx = scmutil.revsingle(repo, opts.get('rev'), None) + if ui.verbose: + tree = fileset.parse(expr) + ui.note(fileset.prettyformat(tree), "\n") + + for f in ctx.getfileset(expr): + ui.write("%s\n" % f) + +@command('debugfsinfo', [], _('[PATH]'), norepo=True) +def debugfsinfo(ui, path="."): + """show information detected about current filesystem""" + util.writefile('.debugfsinfo', '') + ui.write(('exec: %s\n') % (util.checkexec(path) and 'yes' or 'no')) + ui.write(('symlink: %s\n') % (util.checklink(path) and 'yes' or 'no')) + ui.write(('hardlink: %s\n') % (util.checknlink(path) and 'yes' or 'no')) + ui.write(('case-sensitive: %s\n') % (util.fscasesensitive('.debugfsinfo') + and 'yes' or 'no')) + os.unlink('.debugfsinfo') + +@command('debuggetbundle', + [('H', 'head', [], _('id of head node'), _('ID')), + ('C', 'common', [], _('id of common node'), _('ID')), + ('t', 'type', 'bzip2', _('bundle compression type to use'), _('TYPE'))], + _('REPO FILE [-H|-C ID]...'), + norepo=True) +def debuggetbundle(ui, repopath, bundlepath, head=None, common=None, **opts): + """retrieves a bundle from a repo + + Every ID must be a full-length hex node id string. Saves the bundle to the + given file. + """ + repo = hg.peer(ui, opts, repopath) + if not repo.capable('getbundle'): + raise error.Abort("getbundle() not supported by target repository") + args = {} + if common: + args['common'] = [bin(s) for s in common] + if head: + args['heads'] = [bin(s) for s in head] + # TODO: get desired bundlecaps from command line. + args['bundlecaps'] = None + bundle = repo.getbundle('debug', **args) + + bundletype = opts.get('type', 'bzip2').lower() + btypes = {'none': 'HG10UN', + 'bzip2': 'HG10BZ', + 'gzip': 'HG10GZ', + 'bundle2': 'HG20'} + bundletype = btypes.get(bundletype) + if bundletype not in bundle2.bundletypes: + raise error.Abort(_('unknown bundle type specified with --type')) + bundle2.writebundle(ui, bundle, bundlepath, bundletype) + +@command('debugignore', [], '[FILE]') +def debugignore(ui, repo, *files, **opts): + """display the combined ignore pattern and information about ignored files + + With no argument display the combined ignore pattern. + + Given space separated file names, shows if the given file is ignored and + if so, show the ignore rule (file and line number) that matched it. + """ + ignore = repo.dirstate._ignore + if not files: + # Show all the patterns + includepat = getattr(ignore, 'includepat', None) + if includepat is not None: + ui.write("%s\n" % includepat) + else: + raise error.Abort(_("no ignore patterns found")) + else: + for f in files: + nf = util.normpath(f) + ignored = None + ignoredata = None + if nf != '.': + if ignore(nf): + ignored = nf + ignoredata = repo.dirstate._ignorefileandline(nf) + else: + for p in util.finddirs(nf): + if ignore(p): + ignored = p + ignoredata = repo.dirstate._ignorefileandline(p) + break + if ignored: + if ignored == nf: + ui.write(_("%s is ignored\n") % f) + else: + ui.write(_("%s is ignored because of " + "containing folder %s\n") + % (f, ignored)) + ignorefile, lineno, line = ignoredata + ui.write(_("(ignore rule in %s, line %d: '%s')\n") + % (ignorefile, lineno, line)) + else: + ui.write(_("%s is not ignored\n") % f) + +@command('debugindex', commands.debugrevlogopts + + [('f', 'format', 0, _('revlog format'), _('FORMAT'))], + _('[-f FORMAT] -c|-m|FILE'), + optionalrepo=True) +def debugindex(ui, repo, file_=None, **opts): + """dump the contents of an index file""" + r = cmdutil.openrevlog(repo, 'debugindex', file_, opts) + format = opts.get('format', 0) + if format not in (0, 1): + raise error.Abort(_("unknown format %d") % format) + + generaldelta = r.version & revlog.REVLOGGENERALDELTA + if generaldelta: + basehdr = ' delta' + else: + basehdr = ' base' + + if ui.debugflag: + shortfn = hex + else: + shortfn = short + + # There might not be anything in r, so have a sane default + idlen = 12 + for i in r: + idlen = len(shortfn(r.node(i))) + break + + if format == 0: + ui.write((" rev offset length " + basehdr + " linkrev" + " %s %s p2\n") % ("nodeid".ljust(idlen), "p1".ljust(idlen))) + elif format == 1: + ui.write((" rev flag offset length" + " size " + basehdr + " link p1 p2" + " %s\n") % "nodeid".rjust(idlen)) + + for i in r: + node = r.node(i) + if generaldelta: + base = r.deltaparent(i) + else: + base = r.chainbase(i) + if format == 0: + try: + pp = r.parents(node) + except Exception: + pp = [nullid, nullid] + ui.write("% 6d % 9d % 7d % 6d % 7d %s %s %s\n" % ( + i, r.start(i), r.length(i), base, r.linkrev(i), + shortfn(node), shortfn(pp[0]), shortfn(pp[1]))) + elif format == 1: + pr = r.parentrevs(i) + ui.write("% 6d %04x % 8d % 8d % 8d % 6d % 6d % 6d % 6d %s\n" % ( + i, r.flags(i), r.start(i), r.length(i), r.rawsize(i), + base, r.linkrev(i), pr[0], pr[1], shortfn(node))) + +@command('debugindexdot', commands.debugrevlogopts, + _('-c|-m|FILE'), optionalrepo=True) +def debugindexdot(ui, repo, file_=None, **opts): + """dump an index DAG as a graphviz dot file""" + r = cmdutil.openrevlog(repo, 'debugindexdot', file_, opts) + ui.write(("digraph G {\n")) + for i in r: + node = r.node(i) + pp = r.parents(node) + ui.write("\t%d -> %d\n" % (r.rev(pp[0]), i)) + if pp[1] != nullid: + ui.write("\t%d -> %d\n" % (r.rev(pp[1]), i)) + ui.write("}\n") + +@command('debugupgraderepo', [ + ('o', 'optimize', [], _('extra optimization to perform'), _('NAME')), + ('', 'run', False, _('performs an upgrade')), +]) +def debugupgraderepo(ui, repo, run=False, optimize=None): + """upgrade a repository to use different features + + If no arguments are specified, the repository is evaluated for upgrade + and a list of problems and potential optimizations is printed. + + With ``--run``, a repository upgrade is performed. Behavior of the upgrade + can be influenced via additional arguments. More details will be provided + by the command output when run without ``--run``. + + During the upgrade, the repository will be locked and no writes will be + allowed. + + At the end of the upgrade, the repository may not be readable while new + repository data is swapped in. This window will be as long as it takes to + rename some directories inside the ``.hg`` directory. On most machines, this + should complete almost instantaneously and the chances of a consumer being + unable to access the repository should be low. + """ + return repair.upgraderepo(ui, repo, run=run, optimize=optimize) diff -Nru mercurial-4.0.2~trusty1/mercurial/destutil.py mercurial-4.1~rc~trusty1/mercurial/destutil.py --- mercurial-4.0.2~trusty1/mercurial/destutil.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/destutil.py 2017-01-18 16:54:51.000000000 +0000 @@ -133,7 +133,7 @@ assert node is not None, "'tip' exists even in empty repository" return node, movemark, None -# order in which each step should be evalutated +# order in which each step should be evaluated # steps are run until one finds a destination destupdatesteps = ['evolution', 'bookmark', 'branch', 'branchfallback'] # mapping to ease extension overriding steps. diff -Nru mercurial-4.0.2~trusty1/mercurial/dirstateguard.py mercurial-4.1~rc~trusty1/mercurial/dirstateguard.py --- mercurial-4.0.2~trusty1/mercurial/dirstateguard.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/dirstateguard.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,69 @@ +# dirstateguard.py - class to allow restoring dirstate after failure +# +# Copyright 2005-2007 Matt Mackall +# +# This software may be used and distributed according to the terms of the +# GNU General Public License version 2 or any later version. + +from __future__ import absolute_import + +from .i18n import _ + +from . import ( + error, +) + +class dirstateguard(object): + '''Restore dirstate at unexpected failure. + + At the construction, this class does: + + - write current ``repo.dirstate`` out, and + - save ``.hg/dirstate`` into the backup file + + This restores ``.hg/dirstate`` from backup file, if ``release()`` + is invoked before ``close()``. + + This just removes the backup file at ``close()`` before ``release()``. + ''' + + def __init__(self, repo, name): + self._repo = repo + self._active = False + self._closed = False + self._suffix = '.backup.%s.%d' % (name, id(self)) + repo.dirstate.savebackup(repo.currenttransaction(), self._suffix) + self._active = True + + def __del__(self): + if self._active: # still active + # this may occur, even if this class is used correctly: + # for example, releasing other resources like transaction + # may raise exception before ``dirstateguard.release`` in + # ``release(tr, ....)``. + self._abort() + + def close(self): + if not self._active: # already inactivated + msg = (_("can't close already inactivated backup: dirstate%s") + % self._suffix) + raise error.Abort(msg) + + self._repo.dirstate.clearbackup(self._repo.currenttransaction(), + self._suffix) + self._active = False + self._closed = True + + def _abort(self): + self._repo.dirstate.restorebackup(self._repo.currenttransaction(), + self._suffix) + self._active = False + + def release(self): + if not self._closed: + if not self._active: # already inactivated + msg = (_("can't release already inactivated backup:" + " dirstate%s") + % self._suffix) + raise error.Abort(msg) + self._abort() diff -Nru mercurial-4.0.2~trusty1/mercurial/dirstate.py mercurial-4.1~rc~trusty1/mercurial/dirstate.py --- mercurial-4.0.2~trusty1/mercurial/dirstate.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/dirstate.py 2017-01-18 16:54:51.000000000 +0000 @@ -21,6 +21,7 @@ osutil, parsers, pathutil, + pycompat, scmutil, util, ) @@ -66,7 +67,7 @@ This returns '(fp, is_pending_opened)' tuple. ''' - if root == os.environ.get('HG_PENDING'): + if root == encoding.environ.get('HG_PENDING'): try: return (vfs('%s.pending' % filename), True) except IOError as inst: @@ -215,7 +216,7 @@ @propertycache def _slash(self): - return self._ui.configbool('ui', 'slash') and os.sep != '/' + return self._ui.configbool('ui', 'slash') and pycompat.ossep != '/' @propertycache def _checklink(self): @@ -270,7 +271,7 @@ @propertycache def _cwd(self): - return os.getcwd() + return pycompat.getcwd() def getcwd(self): '''Return the path from which a canonical path is calculated. @@ -285,7 +286,7 @@ # self._root ends with a path separator if self._root is '/' or 'C:\' rootsep = self._root if not util.endswithsep(rootsep): - rootsep += os.sep + rootsep += pycompat.ossep if cwd.startswith(rootsep): return cwd[len(rootsep):] else: diff -Nru mercurial-4.0.2~trusty1/mercurial/dispatch.py mercurial-4.1~rc~trusty1/mercurial/dispatch.py --- mercurial-4.0.2~trusty1/mercurial/dispatch.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/dispatch.py 2017-01-18 16:54:51.000000000 +0000 @@ -10,12 +10,11 @@ import atexit import difflib import errno +import getopt import os import pdb import re -import shlex import signal -import socket import sys import time import traceback @@ -25,7 +24,9 @@ from . import ( cmdutil, + color, commands, + debugcommands, demandimport, encoding, error, @@ -35,7 +36,9 @@ hg, hook, profiling, + pycompat, revset, + scmutil, templatefilters, templatekw, templater, @@ -57,7 +60,7 @@ def run(): "run the command in sys.argv" - sys.exit((dispatch(request(sys.argv[1:])) or 0) & 255) + sys.exit((dispatch(request(pycompat.sysargv[1:])) or 0) & 255) def _getsimilar(symbols, value): sim = lambda x: difflib.SequenceMatcher(None, value, x).ratio() @@ -95,11 +98,11 @@ elif req.ui: ferr = req.ui.ferr else: - ferr = sys.stderr + ferr = util.stderr try: if not req.ui: - req.ui = uimod.ui() + req.ui = uimod.ui.load() if '--traceback' in req.args: req.ui.setconfig('ui', 'traceback', 'on', '--traceback') @@ -216,30 +219,15 @@ return callcatch(ui, _runcatchfunc) def callcatch(ui, func): - """call func() with global exception handling - - return func() if no exception happens. otherwise do some error handling - and return an exit code accordingly. + """like scmutil.callcatch but handles more high-level exceptions about + config parsing and commands. besides, use handlecommandexception to handle + uncaught exceptions. """ try: - return func() - # Global exception handling, alphabetically - # Mercurial-specific first, followed by built-in and library exceptions + return scmutil.callcatch(ui, func) except error.AmbiguousCommand as inst: ui.warn(_("hg: command '%s' is ambiguous:\n %s\n") % (inst.args[0], " ".join(inst.args[1]))) - except error.ParseError as inst: - _formatparse(ui.warn, inst) - return -1 - except error.LockHeld as inst: - if inst.errno == errno.ETIMEDOUT: - reason = _('timed out waiting for lock held by %s') % inst.locker - else: - reason = _('lock held by %s') % inst.locker - ui.warn(_("abort: %s: %s\n") % (inst.desc or inst.filename, reason)) - except error.LockUnavailable as inst: - ui.warn(_("abort: could not lock %s: %s\n") % - (inst.desc or inst.filename, inst.strerror)) except error.CommandError as inst: if inst.args[0]: ui.warn(_("hg %s: %s\n") % (inst.args[0], inst.args[1])) @@ -247,34 +235,9 @@ else: ui.warn(_("hg: %s\n") % inst.args[1]) commands.help_(ui, 'shortlist') - except error.OutOfBandError as inst: - if inst.args: - msg = _("abort: remote error:\n") - else: - msg = _("abort: remote error\n") - ui.warn(msg) - if inst.args: - ui.warn(''.join(inst.args)) - if inst.hint: - ui.warn('(%s)\n' % inst.hint) - except error.RepoError as inst: - ui.warn(_("abort: %s!\n") % inst) - if inst.hint: - ui.warn(_("(%s)\n") % inst.hint) - except error.ResponseError as inst: - ui.warn(_("abort: %s") % inst.args[0]) - if not isinstance(inst.args[1], basestring): - ui.warn(" %r\n" % (inst.args[1],)) - elif not inst.args[1]: - ui.warn(_(" empty string\n")) - else: - ui.warn("\n%r\n" % util.ellipsis(inst.args[1])) - except error.CensoredNodeError as inst: - ui.warn(_("abort: file censored %s!\n") % inst) - except error.RevlogError as inst: - ui.warn(_("abort: %s!\n") % inst) - except error.SignalInterrupt: - ui.warn(_("killed!\n")) + except error.ParseError as inst: + _formatparse(ui.warn, inst) + return -1 except error.UnknownCommand as inst: ui.warn(_("hg: unknown command '%s'\n") % inst.args[0]) try: @@ -290,61 +253,11 @@ suggested = True if not suggested: commands.help_(ui, 'shortlist') - except error.InterventionRequired as inst: - ui.warn("%s\n" % inst) - if inst.hint: - ui.warn(_("(%s)\n") % inst.hint) - return 1 - except error.Abort as inst: - ui.warn(_("abort: %s\n") % inst) - if inst.hint: - ui.warn(_("(%s)\n") % inst.hint) - except ImportError as inst: - ui.warn(_("abort: %s!\n") % inst) - m = str(inst).split()[-1] - if m in "mpatch bdiff".split(): - ui.warn(_("(did you forget to compile extensions?)\n")) - elif m in "zlib".split(): - ui.warn(_("(is your Python install correct?)\n")) - except IOError as inst: - if util.safehasattr(inst, "code"): - ui.warn(_("abort: %s\n") % inst) - elif util.safehasattr(inst, "reason"): - try: # usually it is in the form (errno, strerror) - reason = inst.reason.args[1] - except (AttributeError, IndexError): - # it might be anything, for example a string - reason = inst.reason - if isinstance(reason, unicode): - # SSLError of Python 2.7.9 contains a unicode - reason = reason.encode(encoding.encoding, 'replace') - ui.warn(_("abort: error: %s\n") % reason) - elif (util.safehasattr(inst, "args") - and inst.args and inst.args[0] == errno.EPIPE): - pass - elif getattr(inst, "strerror", None): - if getattr(inst, "filename", None): - ui.warn(_("abort: %s: %s\n") % (inst.strerror, inst.filename)) - else: - ui.warn(_("abort: %s\n") % inst.strerror) - else: - raise - except OSError as inst: - if getattr(inst, "filename", None) is not None: - ui.warn(_("abort: %s: '%s'\n") % (inst.strerror, inst.filename)) - else: - ui.warn(_("abort: %s\n") % inst.strerror) + except IOError: + raise except KeyboardInterrupt: raise - except MemoryError: - ui.warn(_("abort: out of memory\n")) - except SystemExit as inst: - # Commands shouldn't sys.exit directly, but give a return code. - # Just in case catch this and and pass exit code to caller. - return inst.code - except socket.error as inst: - ui.warn(_("abort: %s\n") % inst.args[-1]) - except: # perhaps re-raises + except: # probably re-raises if not handlecommandexception(ui): raise @@ -365,7 +278,7 @@ cmd = re.sub(r'\$(\d+|\$)', replacer, cmd) givenargs = [x for i, x in enumerate(givenargs) if i not in nums] - args = shlex.split(cmd) + args = pycompat.shlexsplit(cmd) return args + givenargs def aliasinterpolate(name, args, cmd): @@ -437,7 +350,7 @@ return try: - args = shlex.split(self.definition) + args = pycompat.shlexsplit(self.definition) except ValueError as inst: self.badalias = (_("error in definition for alias '%s': %s") % (self.name, inst)) @@ -536,7 +449,7 @@ try: args = fancyopts.fancyopts(args, commands.globalopts, options) - except fancyopts.getopt.GetoptError as inst: + except getopt.GetoptError as inst: raise error.CommandError(None, inst) if args: @@ -547,7 +460,7 @@ args = aliasargs(entry[0], args) defaults = ui.config("defaults", cmd) if defaults: - args = map(util.expandpath, shlex.split(defaults)) + args + args = map(util.expandpath, pycompat.shlexsplit(defaults)) + args c = list(entry[1]) else: cmd = None @@ -559,7 +472,7 @@ try: args = fancyopts.fancyopts(args, c, cmdoptions, gnu=True) - except fancyopts.getopt.GetoptError as inst: + except getopt.GetoptError as inst: raise error.CommandError(cmd, inst) # separate global options back out @@ -665,7 +578,7 @@ """ if wd is None: try: - wd = os.getcwd() + wd = pycompat.getcwd() except OSError as e: raise error.Abort(_("error getting current working directory: %s") % e.strerror) @@ -689,7 +602,7 @@ try: args = fancyopts.fancyopts(args, commands.globalopts, options) - except fancyopts.getopt.GetoptError: + except getopt.GetoptError: return if not args: @@ -712,14 +625,6 @@ return lambda: runcommand(lui, None, cmd, args[:1], ui, options, d, [], {}) -def _cmdattr(ui, cmd, func, attr): - try: - return getattr(func, attr) - except AttributeError: - ui.deprecwarn("missing attribute '%s', use @command decorator " - "to register '%s'" % (attr, cmd), '3.8') - return False - _loaded = set() # list of (objname, loadermod, loadername) tuple: @@ -730,6 +635,7 @@ # extraobj) arguments extraloaders = [ ('cmdtable', commands, 'loadcmdtable'), + ('colortable', color, 'loadcolortable'), ('filesetpredicate', fileset, 'loadpredicate'), ('revsetpredicate', revset, 'loadpredicate'), ('templatefilter', templatefilters, 'loadfilter'), @@ -768,6 +674,10 @@ # (reposetup is handled in hg.repository) + # Side-effect of accessing is debugcommands module is guaranteed to be + # imported and commands.table is populated. + debugcommands.command + addaliases(lui, commands.table) # All aliases and commands are completely defined, now. @@ -848,7 +758,7 @@ with profiling.maybeprofile(lui): repo = None cmdpats = args[:] - if not _cmdattr(ui, cmd, func, 'norepo'): + if not func.norepo: # use the repo from the request only if we don't have -R if not rpath and not cwd: repo = req.repo @@ -871,9 +781,8 @@ except error.RepoError: if rpath and rpath[-1]: # invalid -R path raise - if not _cmdattr(ui, cmd, func, 'optionalrepo'): - if (_cmdattr(ui, cmd, func, 'inferrepo') and - args and not path): + if not func.optionalrepo: + if func.inferrepo and args and not path: # try to infer -R from command args repos = map(cmdutil.findrepo, args) guess = repos[0] @@ -883,7 +792,7 @@ if not path: raise error.RepoError(_("no repository found in" " '%s' (.hg not found)") - % os.getcwd()) + % pycompat.getcwd()) raise if repo: ui = repo.ui @@ -895,7 +804,8 @@ msg = ' '.join(' ' in a and repr(a) or a for a in fullargs) ui.log("command", '%s\n', msg) - d = lambda: util.checksignature(func)(ui, *args, **cmdoptions) + strcmdopt = pycompat.strkwargs(cmdoptions) + d = lambda: util.checksignature(func)(ui, *args, **strcmdopt) try: return runcommand(lui, repo, cmd, fullargs, ui, options, d, cmdpats, cmdoptions) diff -Nru mercurial-4.0.2~trusty1/mercurial/encoding.py mercurial-4.1~rc~trusty1/mercurial/encoding.py --- mercurial-4.0.2~trusty1/mercurial/encoding.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/encoding.py 2017-01-18 16:54:51.000000000 +0000 @@ -93,7 +93,7 @@ try: encoding = environ.get("HGENCODING") if not encoding: - encoding = locale.getpreferredencoding() or 'ascii' + encoding = locale.getpreferredencoding().encode('ascii') or 'ascii' encoding = _encodingfixers.get(encoding, lambda: encoding)() except locale.Error: encoding = 'ascii' diff -Nru mercurial-4.0.2~trusty1/mercurial/error.py mercurial-4.1~rc~trusty1/mercurial/error.py --- mercurial-4.0.2~trusty1/mercurial/error.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/error.py 2017-01-18 16:54:51.000000000 +0000 @@ -84,7 +84,7 @@ """Raised when an update is aborted because there is nothing to merge""" class ManyMergeDestAbort(MergeDestAbort): - """Raised when an update is aborted because destination is ambigious""" + """Raised when an update is aborted because destination is ambiguous""" class ResponseExpected(Abort): """Raised when an EOF is received for a prompt""" @@ -168,6 +168,9 @@ class PushRaced(RuntimeError): """An exception raised during unbundling that indicate a push race""" +class ProgrammingError(RuntimeError): + """Raised if a mercurial (core or extension) developer made a mistake""" + # bundle2 related errors class BundleValueError(ValueError): """error raised when bundle2 cannot be processed""" diff -Nru mercurial-4.0.2~trusty1/mercurial/exchange.py mercurial-4.1~rc~trusty1/mercurial/exchange.py --- mercurial-4.0.2~trusty1/mercurial/exchange.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/exchange.py 2017-01-18 16:54:51.000000000 +0000 @@ -37,12 +37,6 @@ urlerr = util.urlerr urlreq = util.urlreq -# Maps bundle compression human names to internal representation. -_bundlespeccompressions = {'none': None, - 'bzip2': 'BZ', - 'gzip': 'GZ', - } - # Maps bundle version human names to changegroup versions. _bundlespeccgversions = {'v1': '01', 'v2': '02', @@ -64,7 +58,7 @@ Where is one of the supported compression formats and is (currently) a version string. A ";" can follow the type and - all text afterwards is interpretted as URI encoded, ";" delimited key=value + all text afterwards is interpreted as URI encoded, ";" delimited key=value pairs. If ``strict`` is True (the default) is required. Otherwise, @@ -114,7 +108,7 @@ if '-' in spec: compression, version = spec.split('-', 1) - if compression not in _bundlespeccompressions: + if compression not in util.compengines.supportedbundlenames: raise error.UnsupportedBundleSpecification( _('%s compression is not supported') % compression) @@ -130,7 +124,7 @@ spec, params = parseparams(spec) - if spec in _bundlespeccompressions: + if spec in util.compengines.supportedbundlenames: compression = spec version = 'v1' if 'generaldelta' in repo.requirements: @@ -157,7 +151,8 @@ ', '.join(sorted(missingreqs))) if not externalnames: - compression = _bundlespeccompressions[compression] + engine = util.compengines.forbundlename(compression) + compression = engine.bundletype()[1] version = _bundlespeccgversions[version] return compression, version, params @@ -196,10 +191,10 @@ restored. """ def speccompression(alg): - for k, v in _bundlespeccompressions.items(): - if v == alg: - return k - return None + try: + return util.compengines.forbundletype(alg).bundletype()[0] + except KeyError: + return None b = readbundle(ui, fh, None) if isinstance(b, changegroup.cg1unpacker): @@ -282,7 +277,7 @@ This function is used to allow testing of the older bundle version""" ui = op.repo.ui forcebundle1 = False - # The goal is this config is to allow developper to choose the bundle + # The goal is this config is to allow developer to choose the bundle # version used during exchanged. This is especially handy during test. # Value is a list of bundle version to be picked from, highest version # should be used. @@ -608,8 +603,21 @@ explicit = set([repo._bookmarks.expandname(bookmark) for bookmark in pushop.bookmarks]) - comp = bookmod.compare(repo, repo._bookmarks, remotebookmark, srchex=hex) + remotebookmark = bookmod.unhexlifybookmarks(remotebookmark) + comp = bookmod.comparebookmarks(repo, repo._bookmarks, remotebookmark) + + def safehex(x): + if x is None: + return x + return hex(x) + + def hexifycompbookmarks(bookmarks): + for b, scid, dcid in bookmarks: + yield b, safehex(scid), safehex(dcid) + + comp = [hexifycompbookmarks(marks) for marks in comp] addsrc, adddst, advsrc, advdst, diverge, differ, invalid, same = comp + for b, scid, dcid in advsrc: if b in explicit: explicit.remove(b) @@ -621,7 +629,7 @@ explicit.remove(b) pushop.outbookmarks.append((b, '', scid)) # search for overwritten bookmark - for b, scid, dcid in advdst + diverge + differ: + for b, scid, dcid in list(advdst) + list(diverge) + list(differ): if b in explicit: explicit.remove(b) pushop.outbookmarks.append((b, dcid, scid)) @@ -1425,7 +1433,7 @@ pullop.stepsdone.add('phases') publishing = bool(remotephases.get('publishing', False)) if remotephases and not publishing: - # remote is new and unpublishing + # remote is new and non-publishing pheads, _dr = phases.analyzeremotephases(pullop.repo, pullop.pulledsubset, remotephases) @@ -1460,6 +1468,7 @@ pullop.stepsdone.add('bookmarks') repo = pullop.repo remotebookmarks = pullop.remotebookmarks + remotebookmarks = bookmod.unhexlifybookmarks(remotebookmarks) bookmod.updatefromremote(repo.ui, repo, remotebookmarks, pullop.remote.url(), pullop.gettransaction, @@ -1666,6 +1675,17 @@ if chunks: bundler.newpart('hgtagsfnodes', data=''.join(chunks)) +def _getbookmarks(repo, **kwargs): + """Returns bookmark to node mapping. + + This function is primarily used to generate `bookmarks` bundle2 part. + It is a separate function in order to make it easy to wrap it + in extensions. Passing `kwargs` to the function makes it easy to + add new parameters in extensions. + """ + + return dict(bookmod.listbinbookmarks(repo)) + def check_heads(repo, their_heads, context): """check if the heads of a repo have been modified @@ -1882,18 +1902,22 @@ return newentries -def sortclonebundleentries(ui, entries): - prefers = ui.configlist('ui', 'clonebundleprefers', default=[]) - if not prefers: - return list(entries) +class clonebundleentry(object): + """Represents an item in a clone bundles manifest. - prefers = [p.split('=', 1) for p in prefers] + This rich class is needed to support sorting since sorted() in Python 3 + doesn't support ``cmp`` and our comparison is complex enough that ``key=`` + won't work. + """ - # Our sort function. - def compareentry(a, b): - for prefkey, prefvalue in prefers: - avalue = a.get(prefkey) - bvalue = b.get(prefkey) + def __init__(self, value, prefers): + self.value = value + self.prefers = prefers + + def _cmp(self, other): + for prefkey, prefvalue in self.prefers: + avalue = self.value.get(prefkey) + bvalue = other.value.get(prefkey) # Special case for b missing attribute and a matches exactly. if avalue is not None and bvalue is None and avalue == prefvalue: @@ -1924,7 +1948,33 @@ # back to index order. return 0 - return sorted(entries, cmp=compareentry) + def __lt__(self, other): + return self._cmp(other) < 0 + + def __gt__(self, other): + return self._cmp(other) > 0 + + def __eq__(self, other): + return self._cmp(other) == 0 + + def __le__(self, other): + return self._cmp(other) <= 0 + + def __ge__(self, other): + return self._cmp(other) >= 0 + + def __ne__(self, other): + return self._cmp(other) != 0 + +def sortclonebundleentries(ui, entries): + prefers = ui.configlist('ui', 'clonebundleprefers', default=[]) + if not prefers: + return list(entries) + + prefers = [p.split('=', 1) for p in prefers] + + items = sorted(clonebundleentry(v, prefers) for v in entries) + return [i.value for i in items] def trypullbundlefromurl(ui, repo, url): """Attempt to apply a bundle from a URL.""" diff -Nru mercurial-4.0.2~trusty1/mercurial/extensions.py mercurial-4.1~rc~trusty1/mercurial/extensions.py --- mercurial-4.0.2~trusty1/mercurial/extensions.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/extensions.py 2017-01-18 16:54:51.000000000 +0000 @@ -18,6 +18,7 @@ from . import ( cmdutil, error, + pycompat, util, ) @@ -59,6 +60,8 @@ def loadpath(path, module_name): module_name = module_name.replace('.', '_') path = util.normpath(util.expandpath(path)) + module_name = pycompat.fsdecode(module_name) + path = pycompat.fsdecode(path) if os.path.isdir(path): # module/__init__.py style d, f = os.path.split(path) @@ -74,7 +77,7 @@ def _importh(name): """import and return the module""" - mod = __import__(name) + mod = __import__(pycompat.sysstr(name)) components = name.split('.') for comp in components[1:]: mod = getattr(mod, comp) @@ -426,7 +429,7 @@ file.close() if doc: # extracting localized synopsis - return gettext(doc).splitlines()[0] + return gettext(doc) else: return _('(no help text available)') @@ -448,7 +451,7 @@ for name, path in paths.iteritems(): doc = _disabledhelp(path) if doc: - exts[name] = doc + exts[name] = doc.splitlines()[0] return exts diff -Nru mercurial-4.0.2~trusty1/mercurial/fancyopts.py mercurial-4.1~rc~trusty1/mercurial/fancyopts.py --- mercurial-4.0.2~trusty1/mercurial/fancyopts.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/fancyopts.py 2017-01-18 16:54:51.000000000 +0000 @@ -7,10 +7,11 @@ from __future__ import absolute_import -import getopt - from .i18n import _ -from . import error +from . import ( + error, + pycompat, +) # Set of flags to not apply boolean negation logic on nevernegate = set([ @@ -34,13 +35,14 @@ stopindex = args.index('--') extraargs = args[stopindex + 1:] args = args[:stopindex] - opts, parseargs = getopt.getopt(args, options, longoptions) + opts, parseargs = pycompat.getoptb(args, options, longoptions) args = [] while parseargs: arg = parseargs.pop(0) if arg and arg[0] == '-' and len(arg) > 1: parseargs.insert(0, arg) - topts, newparseargs = getopt.getopt(parseargs, options, longoptions) + topts, newparseargs = pycompat.getoptb(parseargs,\ + options, longoptions) opts = opts + topts parseargs = newparseargs else: @@ -125,7 +127,7 @@ if gnu: parse = gnugetopt else: - parse = getopt.getopt + parse = pycompat.getoptb opts, args = parse(args, shortlist, namelist) # transfer result to state diff -Nru mercurial-4.0.2~trusty1/mercurial/filelog.py mercurial-4.1~rc~trusty1/mercurial/filelog.py --- mercurial-4.0.2~trusty1/mercurial/filelog.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/filelog.py 2017-01-18 16:54:51.000000000 +0000 @@ -104,9 +104,9 @@ return True - def checkhash(self, text, p1, p2, node, rev=None): + def checkhash(self, text, node, p1=None, p2=None, rev=None): try: - super(filelog, self).checkhash(text, p1, p2, node, rev=rev) + super(filelog, self).checkhash(text, node, p1=p1, p2=p2, rev=rev) except error.RevlogError: if _censoredtext(text): raise error.CensoredNodeError(self.indexfile, node, text) diff -Nru mercurial-4.0.2~trusty1/mercurial/filemerge.py mercurial-4.1~rc~trusty1/mercurial/filemerge.py --- mercurial-4.0.2~trusty1/mercurial/filemerge.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/filemerge.py 2017-01-18 16:54:51.000000000 +0000 @@ -16,6 +16,7 @@ from .node import nullid, short from . import ( + encoding, error, formatter, match, @@ -165,7 +166,7 @@ return (force, force) # HGMERGE takes next precedence - hgmerge = os.environ.get("HGMERGE") + hgmerge = encoding.environ.get("HGMERGE") if hgmerge: if changedelete and not supportscd(hgmerge): return ":prompt", None @@ -189,7 +190,8 @@ if _toolbool(ui, t, "disabled", False): disabled.add(t) names = tools.keys() - tools = sorted([(-p, t) for t, p in tools.items() if t not in disabled]) + tools = sorted([(-p, tool) for tool, p in tools.items() + if tool not in disabled]) uimerge = ui.config("ui", "merge") if uimerge: # external tools defined in uimerge won't be able to handle @@ -517,7 +519,8 @@ return util.ellipsis(mark, 80 - 8) _defaultconflictmarker = ('{node|short} ' - '{ifeq(tags, "tip", "", "{tags} ")}' + '{ifeq(tags, "tip", "", ' + 'ifeq(tags, "", "", "{tags} "))}' '{if(bookmarks, "{bookmarks} ")}' '{ifeq(branch, "default", "", "{branch} ")}' '- {author|user}: {desc|firstline}') @@ -575,8 +578,9 @@ a boolean indicating whether the file was deleted from disk.""" def temp(prefix, ctx): - pre = "%s~%s." % (os.path.basename(ctx.path()), prefix) - (fd, name) = tempfile.mkstemp(prefix=pre) + fullbase, ext = os.path.splitext(ctx.path()) + pre = "%s~%s." % (os.path.basename(fullbase), prefix) + (fd, name) = tempfile.mkstemp(prefix=pre, suffix=ext) data = repo.wwritedata(ctx.path(), ctx.data()) f = os.fdopen(fd, "wb") f.write(data) diff -Nru mercurial-4.0.2~trusty1/mercurial/formatter.py mercurial-4.1~rc~trusty1/mercurial/formatter.py --- mercurial-4.0.2~trusty1/mercurial/formatter.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/formatter.py 2017-01-18 16:54:51.000000000 +0000 @@ -5,6 +5,101 @@ # This software may be used and distributed according to the terms of the # GNU General Public License version 2 or any later version. +"""Generic output formatting for Mercurial + +The formatter provides API to show data in various ways. The following +functions should be used in place of ui.write(): + +- fm.write() for unconditional output +- fm.condwrite() to show some extra data conditionally in plain output +- fm.data() to provide extra data to JSON or template output +- fm.plain() to show raw text that isn't provided to JSON or template output + +To show structured data (e.g. date tuples, dicts, lists), apply fm.format*() +beforehand so the data is converted to the appropriate data type. Use +fm.isplain() if you need to convert or format data conditionally which isn't +supported by the formatter API. + +To build nested structure (i.e. a list of dicts), use fm.nested(). + +See also https://www.mercurial-scm.org/wiki/GenericTemplatingPlan + +fm.condwrite() vs 'if cond:': + +In most cases, use fm.condwrite() so users can selectively show the data +in template output. If it's costly to build data, use plain 'if cond:' with +fm.write(). + +fm.nested() vs fm.formatdict() (or fm.formatlist()): + +fm.nested() should be used to form a tree structure (a list of dicts of +lists of dicts...) which can be accessed through template keywords, e.g. +"{foo % "{bar % {...}} {baz % {...}}"}". On the other hand, fm.formatdict() +exports a dict-type object to template, which can be accessed by e.g. +"{get(foo, key)}" function. + +Doctest helper: + +>>> def show(fn, verbose=False, **opts): +... import sys +... from . import ui as uimod +... ui = uimod.ui() +... ui.fout = sys.stdout # redirect to doctest +... ui.verbose = verbose +... return fn(ui, ui.formatter(fn.__name__, opts)) + +Basic example: + +>>> def files(ui, fm): +... files = [('foo', 123, (0, 0)), ('bar', 456, (1, 0))] +... for f in files: +... fm.startitem() +... fm.write('path', '%s', f[0]) +... fm.condwrite(ui.verbose, 'date', ' %s', +... fm.formatdate(f[2], '%Y-%m-%d %H:%M:%S')) +... fm.data(size=f[1]) +... fm.plain('\\n') +... fm.end() +>>> show(files) +foo +bar +>>> show(files, verbose=True) +foo 1970-01-01 00:00:00 +bar 1970-01-01 00:00:01 +>>> show(files, template='json') +[ + { + "date": [0, 0], + "path": "foo", + "size": 123 + }, + { + "date": [1, 0], + "path": "bar", + "size": 456 + } +] +>>> show(files, template='path: {path}\\ndate: {date|rfc3339date}\\n') +path: foo +date: 1970-01-01T00:00:00+00:00 +path: bar +date: 1970-01-01T00:00:01+00:00 + +Nested example: + +>>> def subrepos(ui, fm): +... fm.startitem() +... fm.write('repo', '[%s]\\n', 'baz') +... files(ui, fm.nested('files')) +... fm.end() +>>> show(subrepos) +[baz] +foo +bar +>>> show(subrepos, template='{repo}: {join(files % "{path}", ", ")}\\n') +baz: foo, bar +""" + from __future__ import absolute_import import os diff -Nru mercurial-4.0.2~trusty1/mercurial/hbisect.py mercurial-4.1~rc~trusty1/mercurial/hbisect.py --- mercurial-4.0.2~trusty1/mercurial/hbisect.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/hbisect.py 2017-01-18 16:54:51.000000000 +0000 @@ -98,7 +98,7 @@ tot = len(candidates) unskipped = [c for c in candidates if (c not in skip) and (c != badrev)] if tot == 1 or not unskipped: - return ([changelog.node(rev) for rev in candidates], 0, good) + return ([changelog.node(c) for c in candidates], 0, good) perfect = tot // 2 # find the best node to test diff -Nru mercurial-4.0.2~trusty1/mercurial/help/config.txt mercurial-4.1~rc~trusty1/mercurial/help/config.txt --- mercurial-4.0.2~trusty1/mercurial/help/config.txt 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/mercurial/help/config.txt 2017-01-18 16:54:51.000000000 +0000 @@ -418,6 +418,28 @@ "HG: removed {file}\n" }{if(files, "", "HG: no files changed\n")} +``diff()`` + String: show the diff (see :hg:`help templates` for detail) + +Sometimes it is helpful to show the diff of the changeset in the editor without +having to prefix 'HG: ' to each line so that highlighting works correctly. For +this, Mercurial provides a special string which will ignore everything below +it:: + + HG: ------------------------ >8 ------------------------ + +For example, the template configuration below will show the diff below the +extra message:: + + [committemplate] + changeset = {desc}\n\n + HG: Enter commit message. Lines beginning with 'HG:' are removed. + HG: {extramsg} + HG: ------------------------ >8 ------------------------ + HG: Do not touch the line above. + HG: Everything below will be removed. + {diff()} + .. note:: For some problematic encodings (see :hg:`help win32mbcs` for @@ -1401,7 +1423,7 @@ ``type`` The type of profiler to use. - (default: ls) + (default: stat) ``ls`` Use Python's built-in instrumenting profiler. This profiler @@ -1409,9 +1431,9 @@ first line of a function. This restriction makes it difficult to identify the expensive parts of a non-trivial function. ``stat`` - Use a third-party statistical profiler, statprof. This profiler - currently runs only on Unix systems, and is most useful for - profiling commands that run for longer than about 0.1 seconds. + Use a statistical profiler, statprof. This profiler is most + useful for profiling commands that run for longer than about 0.1 + seconds. ``format`` Profiling format. Specific to the ``ls`` instrumenting profiler. @@ -1426,6 +1448,20 @@ file, the generated file can directly be loaded into kcachegrind. +``statformat`` + Profiling format for the ``stat`` profiler. + (default: hotpath) + + ``hotpath`` + Show a tree-based display containing the hot path of execution (where + most time was spent). + ``bymethod`` + Show a table of methods ordered by how frequently they are active. + ``byline`` + Show a table of lines in files ordered by how frequently they are active. + ``json`` + Render profiling data as JSON. + ``frequency`` Sampling frequency. Specific to the ``stat`` sampling profiler. (default: 1000) @@ -1509,6 +1545,21 @@ Controls generic server settings. +``compressionengines`` + List of compression engines and their relative priority to advertise + to clients. + + The order of compression engines determines their priority, the first + having the highest priority. If a compression engine is not listed + here, it won't be advertised to clients. + + If not set (the default), built-in defaults are used. Run + :hg:`debuginstall` to list available compression engines and their + default wire protocol priority. + + Older Mercurial clients only support zlib compression and this setting + has no effect for legacy clients. + ``uncompressed`` Whether to allow clients to clone a repository using the uncompressed streaming protocol. This transfers about 40% more @@ -1578,6 +1629,18 @@ This option only impacts the HTTP server. +``zstdlevel`` + Integer between ``1`` and ``22`` that controls the zstd compression level + for wire protocol commands. ``1`` is the minimal amount of compression and + ``22`` is the highest amount of compression. + + The default (``3``) should be significantly faster than zlib while likely + delivering better compression ratios. + + This option only impacts the HTTP server. + + See also ``server.zliblevel``. + ``smtp`` -------- @@ -2048,6 +2111,20 @@ Name or email address of the person in charge of the repository. (default: ui.username or ``$EMAIL`` or "unknown" if unset or empty) +``csp`` + Send a ``Content-Security-Policy`` HTTP header with this value. + + The value may contain a special string ``%nonce%``, which will be replaced + by a randomly-generated one-time use value. If the value contains + ``%nonce%``, ``web.cache`` will be disabled, as caching undermines the + one-time property of the nonce. This nonce will also be inserted into + ``
{repo|escape}
- -
test
- @@ -360,7 +358,6 @@ -

Mercurial Repository: test

- @@ -442,7 +440,6 @@
- @@ -579,7 +576,6 @@ - @@ -703,7 +699,6 @@ - @@ -837,7 +832,6 @@ - @@ -969,7 +963,6 @@ - @@ -1096,7 +1089,6 @@ - diff -Nru mercurial-4.0.2~trusty1/tests/test-hgwebdir-paths.py mercurial-4.1~rc~trusty1/tests/test-hgwebdir-paths.py --- mercurial-4.0.2~trusty1/tests/test-hgwebdir-paths.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-hgwebdir-paths.py 2017-01-18 16:54:51.000000000 +0000 @@ -15,7 +15,7 @@ webdir = os.path.realpath('.') -u = uimod.ui() +u = uimod.ui.load() hg.repository(u, 'a', create=1) hg.repository(u, 'b', create=1) os.chdir('b') diff -Nru mercurial-4.0.2~trusty1/tests/test-hgwebdir.t mercurial-4.1~rc~trusty1/tests/test-hgwebdir.t --- mercurial-4.0.2~trusty1/tests/test-hgwebdir.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-hgwebdir.t 2017-01-18 16:54:51.000000000 +0000 @@ -898,7 +898,6 @@ - @@ -969,7 +968,6 @@ - @@ -1407,7 +1405,6 @@ - diff -Nru mercurial-4.0.2~trusty1/tests/test-hgweb-empty.t mercurial-4.1~rc~trusty1/tests/test-hgweb-empty.t --- mercurial-4.0.2~trusty1/tests/test-hgweb-empty.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-hgweb-empty.t 2017-01-18 16:54:51.000000000 +0000 @@ -109,7 +109,6 @@ - @@ -220,7 +219,6 @@ - @@ -374,7 +372,6 @@ - @@ -454,7 +451,6 @@ - diff -Nru mercurial-4.0.2~trusty1/tests/test-hgweb-filelog.t mercurial-4.1~rc~trusty1/tests/test-hgweb-filelog.t --- mercurial-4.0.2~trusty1/tests/test-hgweb-filelog.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-hgweb-filelog.t 2017-01-18 16:54:51.000000000 +0000 @@ -242,7 +242,6 @@ - @@ -362,7 +361,6 @@ - @@ -474,7 +472,6 @@ - @@ -586,7 +583,6 @@ - @@ -651,7 +647,6 @@ - @@ -757,7 +752,6 @@ - - @@ -235,7 +234,6 @@ - diff -Nru mercurial-4.0.2~trusty1/tests/test-hgweb-symrev.t mercurial-4.1~rc~trusty1/tests/test-hgweb-symrev.t --- mercurial-4.0.2~trusty1/tests/test-hgweb-symrev.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-hgweb-symrev.t 2017-01-18 16:54:51.000000000 +0000 @@ -186,7 +186,7 @@
  • diff
  • comparison
  • file log
    • -
  • raw
    • +
  • raw
    • annotate foo @ 1:a7c1559b7bba 43c799df6e75 9d8c40cba617 @@ -381,7 +381,7 @@
  • diff
  • comparison
  • file log
    • -
  • raw
    • +
  • raw
    • annotate foo @ 1:a7c1559b7bba 43c799df6e75 9d8c40cba617 @@ -626,7 +626,7 @@ revisions | diff | comparison | - raw | + raw | a7c1559b7bba 9d8c40cba617 @@ -849,7 +849,7 @@
  • revisions
  • diff
  • comparison
    • -
  • raw
    • +
  • raw
    • a7c1559b7bba
    9d8c40cba617 @@ -1053,7 +1053,7 @@ files file revisions - raw + raw a7c1559b7bba 9d8c40cba617 diff -Nru mercurial-4.0.2~trusty1/tests/test-hgweb.t mercurial-4.1~rc~trusty1/tests/test-hgweb.t --- mercurial-4.0.2~trusty1/tests/test-hgweb.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-hgweb.t 2017-01-18 16:54:51.000000000 +0000 @@ -99,7 +99,6 @@ - @@ -207,7 +206,6 @@ - @@ -318,7 +316,6 @@ - diff -Nru mercurial-4.0.2~trusty1/tests/test-highlight.t mercurial-4.1~rc~trusty1/tests/test-highlight.t --- mercurial-4.0.2~trusty1/tests/test-highlight.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-highlight.t 2017-01-18 16:54:51.000000000 +0000 @@ -186,7 +186,6 @@ - @@ -236,7 +235,7 @@
  • comparison
  • annotate
  • file log
    • -
  • raw
    • +
  • raw
    • help
      • @@ -857,7 +856,6 @@ - diff -Nru mercurial-4.0.2~trusty1/tests/test-histedit-obsolete.t mercurial-4.1~rc~trusty1/tests/test-histedit-obsolete.t --- mercurial-4.0.2~trusty1/tests/test-histedit-obsolete.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-histedit-obsolete.t 2017-01-18 16:54:51.000000000 +0000 @@ -339,7 +339,7 @@ New-commit as draft (default) - $ cp -r base simple-draft + $ cp -R base simple-draft $ cd simple-draft $ hg histedit -r 'b449568bf7fc' --commands - << EOF > edit b449568bf7fc 11 f @@ -378,7 +378,7 @@ New-commit as secret (config) - $ cp -r base simple-secret + $ cp -R base simple-secret $ cd simple-secret $ cat >> .hg/hgrc << EOF > [phases] @@ -425,7 +425,7 @@ If a secret changeset is put before a draft one, all descendant should be secret. It seems more important to present the secret phase. - $ cp -r base reorder + $ cp -R base reorder $ cd reorder $ hg histedit -r 'b449568bf7fc' --commands - << EOF > pick b449568bf7fc 11 f @@ -462,7 +462,7 @@ Note that there is a few reordering in this series for more extensive test - $ cp -r base folding + $ cp -R base folding $ cd folding $ cat >> .hg/hgrc << EOF > [phases] diff -Nru mercurial-4.0.2~trusty1/tests/test-http-branchmap.t mercurial-4.1~rc~trusty1/tests/test-http-branchmap.t --- mercurial-4.0.2~trusty1/tests/test-http-branchmap.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-http-branchmap.t 2017-01-18 16:54:51.000000000 +0000 @@ -81,7 +81,7 @@ > sys.stdout = StdoutWrapper(sys.stdout) > sys.stderr = StdoutWrapper(sys.stderr) > - > myui = ui.ui() + > myui = ui.ui.load() > repo = hg.repository(myui, 'a') > commands.serve(myui, repo, stdio=True, cmdserver=False) > EOF diff -Nru mercurial-4.0.2~trusty1/tests/test-http-bundle1.t mercurial-4.1~rc~trusty1/tests/test-http-bundle1.t --- mercurial-4.0.2~trusty1/tests/test-http-bundle1.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-http-bundle1.t 2017-01-18 16:54:51.000000000 +0000 @@ -238,66 +238,66 @@ $ sed 's/.*] "/"/' < ../access.log "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=stream_out HTTP/1.1" 401 - - "GET /?cmd=stream_out HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D5fed3813f7f5e1824344fdc9cf8f63bb662c292d - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D - "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 403 - x-hgarg-1:namespace=namespaces - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D7f4e523d01f2cc3765ac8934da3d14db775ff872 - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=phases - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=stream_out HTTP/1.1" 401 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=stream_out HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D5fed3813f7f5e1824344fdc9cf8f63bb662c292d x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:common=0000000000000000000000000000000000000000&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 403 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D7f4e523d01f2cc3765ac8934da3d14db775ff872 x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) "POST /?cmd=unbundle HTTP/1.1" 200 - x-hgarg-1:heads=686173686564+5eb5abfefeea63c80dd7553bcc3783f37e0c5524* (glob) - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-http-protocol.t mercurial-4.1~rc~trusty1/tests/test-http-protocol.t --- mercurial-4.0.2~trusty1/tests/test-http-protocol.t 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-http-protocol.t 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,167 @@ + $ cat >> $HGRCPATH << EOF + > [web] + > push_ssl = false + > allow_push = * + > EOF + + $ hg init server + $ cd server + $ touch a + $ hg -q commit -A -m initial + $ cd .. + + $ hg -R server serve -p $HGPORT -d --pid-file hg.pid + $ cat hg.pid >> $DAEMON_PIDS + +compression formats are advertised in compression capability + +#if zstd + $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression + compression=zstd,zlib +#else + $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression + compression=zlib +#endif + + $ killdaemons.py + +server.compressionengines can replace engines list wholesale + + $ hg --config server.compressionengines=none -R server serve -p $HGPORT -d --pid-file hg.pid + $ cat hg.pid > $DAEMON_PIDS + $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression + compression=none + + $ killdaemons.py + +Order of engines can also change + + $ hg --config server.compressionengines=none,zlib -R server serve -p $HGPORT -d --pid-file hg.pid + $ cat hg.pid > $DAEMON_PIDS + $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=capabilities' | tr ' ' '\n' | grep compression + compression=none,zlib + + $ killdaemons.py + +Start a default server again + + $ hg -R server serve -p $HGPORT -d --pid-file hg.pid + $ cat hg.pid > $DAEMON_PIDS + +Server should send application/mercurial-0.1 to clients if no Accept is used + + $ get-with-headers.py --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' - + 200 Script output follows + content-type: application/mercurial-0.1 + date: * (glob) + server: * (glob) + transfer-encoding: chunked + +Server should send application/mercurial-0.1 when client says it wants it + + $ get-with-headers.py --hgproto '0.1' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' - + 200 Script output follows + content-type: application/mercurial-0.1 + date: * (glob) + server: * (glob) + transfer-encoding: chunked + +Server should send application/mercurial-0.2 when client says it wants it + + $ get-with-headers.py --hgproto '0.2' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' - + 200 Script output follows + content-type: application/mercurial-0.2 + date: * (glob) + server: * (glob) + transfer-encoding: chunked + + $ get-with-headers.py --hgproto '0.1 0.2' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' - + 200 Script output follows + content-type: application/mercurial-0.2 + date: * (glob) + server: * (glob) + transfer-encoding: chunked + +Requesting a compression format that server doesn't support results will fall back to 0.1 + + $ get-with-headers.py --hgproto '0.2 comp=aa' --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' - + 200 Script output follows + content-type: application/mercurial-0.1 + date: * (glob) + server: * (glob) + transfer-encoding: chunked + +#if zstd +zstd is used if available + + $ get-with-headers.py --hgproto '0.2 comp=zstd' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp + $ f --size --hexdump --bytes 36 --sha1 resp + resp: size=248, sha1=4d8d8f87fb82bd542ce52881fdc94f850748 + 0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu| + 0010: 74 20 66 6f 6c 6c 6f 77 73 0a 0a 04 7a 73 74 64 |t follows...zstd| + 0020: 28 b5 2f fd |(./.| + +#endif + +application/mercurial-0.2 is not yet used on non-streaming responses + + $ get-with-headers.py --hgproto '0.2' 127.0.0.1:$HGPORT '?cmd=heads' - + 200 Script output follows + content-length: 41 + content-type: application/mercurial-0.1 + date: * (glob) + server: * (glob) + + e93700bd72895c5addab234c56d4024b487a362f + +Now test protocol preference usage + + $ killdaemons.py + $ hg --config server.compressionengines=none,zlib -R server serve -p $HGPORT -d --pid-file hg.pid + $ cat hg.pid > $DAEMON_PIDS + +No Accept will send 0.1+zlib, even though "none" is preferred b/c "none" isn't supported on 0.1 + + $ get-with-headers.py --headeronly 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' Content-Type + 200 Script output follows + content-type: application/mercurial-0.1 + + $ get-with-headers.py 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp + $ f --size --hexdump --bytes 28 --sha1 resp + resp: size=227, sha1=35a4c074da74f32f5440da3cbf04 + 0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu| + 0010: 74 20 66 6f 6c 6c 6f 77 73 0a 0a 78 |t follows..x| + +Explicit 0.1 will send zlib because "none" isn't supported on 0.1 + + $ get-with-headers.py --hgproto '0.1' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp + $ f --size --hexdump --bytes 28 --sha1 resp + resp: size=227, sha1=35a4c074da74f32f5440da3cbf04 + 0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu| + 0010: 74 20 66 6f 6c 6c 6f 77 73 0a 0a 78 |t follows..x| + +0.2 with no compression will get "none" because that is server's preference +(spec says ZL and UN are implicitly supported) + + $ get-with-headers.py --hgproto '0.2' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp + $ f --size --hexdump --bytes 32 --sha1 resp + resp: size=432, sha1=ac931b412ec185a02e0e5bcff98dac83 + 0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu| + 0010: 74 20 66 6f 6c 6c 6f 77 73 0a 0a 04 6e 6f 6e 65 |t follows...none| + +Client receives server preference even if local order doesn't match + + $ get-with-headers.py --hgproto '0.2 comp=zlib,none' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp + $ f --size --hexdump --bytes 32 --sha1 resp + resp: size=432, sha1=ac931b412ec185a02e0e5bcff98dac83 + 0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu| + 0010: 74 20 66 6f 6c 6c 6f 77 73 0a 0a 04 6e 6f 6e 65 |t follows...none| + +Client receives only supported format even if not server preferred format + + $ get-with-headers.py --hgproto '0.2 comp=zlib' 127.0.0.1:$HGPORT '?cmd=getbundle&heads=e93700bd72895c5addab234c56d4024b487a362f&common=0000000000000000000000000000000000000000' > resp + $ f --size --hexdump --bytes 33 --sha1 resp + resp: size=232, sha1=a1c727f0c9693ca15742a75c30419bc36 + 0000: 32 30 30 20 53 63 72 69 70 74 20 6f 75 74 70 75 |200 Script outpu| + 0010: 74 20 66 6f 6c 6c 6f 77 73 0a 0a 04 7a 6c 69 62 |t follows...zlib| + 0020: 78 |x| diff -Nru mercurial-4.0.2~trusty1/tests/test-http-proxy.t mercurial-4.1~rc~trusty1/tests/test-http-proxy.t --- mercurial-4.0.2~trusty1/tests/test-http-proxy.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-http-proxy.t 2017-01-18 16:54:51.000000000 +0000 @@ -101,19 +101,19 @@ 1 files updated, 0 files merged, 0 files removed, 0 files unresolved $ cat proxy.log * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=branchmap HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=stream_out HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D83180e7845de420a1bb46896fd5fe05294f8d629 (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=83180e7845de420a1bb46896fd5fe05294f8d629&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=branchmap HTTP/1.1" - - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=stream_out HTTP/1.1" - - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D83180e7845de420a1bb46896fd5fe05294f8d629 x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=83180e7845de420a1bb46896fd5fe05294f8d629&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET http://localhost:$HGPORT/?cmd=capabilities HTTP/1.1" - - (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D (glob) - * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=batch HTTP/1.1" - - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET http://localhost:$HGPORT/?cmd=getbundle HTTP/1.1" - - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=83180e7845de420a1bb46896fd5fe05294f8d629&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) diff -Nru mercurial-4.0.2~trusty1/tests/test-https.t mercurial-4.1~rc~trusty1/tests/test-https.t --- mercurial-4.0.2~trusty1/tests/test-https.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-https.t 2017-01-18 16:54:51.000000000 +0000 @@ -379,7 +379,7 @@ Fingerprints -- works without cacerts (hostkeyfingerprints) +- works without cacerts (hostfingerprints) $ hg -R copy-pull id https://localhost:$HGPORT/ --insecure --config hostfingerprints.localhost=ec:d8:7c:d6:b3:86:d0:4f:c1:b8:b4:1c:9d:8f:5e:16:8e:ef:1c:03 warning: connecting to localhost using legacy security technology (TLS 1.0); see https://mercurial-scm.org/wiki/SecureConnections for more info (?) 5fed3813f7f5 diff -Nru mercurial-4.0.2~trusty1/tests/test-http.t mercurial-4.1~rc~trusty1/tests/test-http.t --- mercurial-4.0.2~trusty1/tests/test-http.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-http.t 2017-01-18 16:54:51.000000000 +0000 @@ -229,63 +229,63 @@ $ sed 's/.*] "/"/' < ../access.log "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=stream_out HTTP/1.1" 401 - - "GET /?cmd=stream_out HTTP/1.1" 200 - - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D5fed3813f7f5e1824344fdc9cf8f63bb662c292d - "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&listkeys=phases%2Cbookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D - "GET /?cmd=getbundle HTTP/1.1" 401 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&listkeys=phases%2Cbookmarks - "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&listkeys=phases%2Cbookmarks - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces - "GET /?cmd=listkeys HTTP/1.1" 403 - x-hgarg-1:namespace=namespaces - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D7f4e523d01f2cc3765ac8934da3d14db775ff872 - "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=phases - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=stream_out HTTP/1.1" 401 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=stream_out HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D5fed3813f7f5e1824344fdc9cf8f63bb662c292d x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=0&common=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=getbundle HTTP/1.1" 401 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=0000000000000000000000000000000000000000&heads=5fed3813f7f5e1824344fdc9cf8f63bb662c292d&listkeys=phases%2Cbookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=lookup HTTP/1.1" 200 - x-hgarg-1:key=tip x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 403 - x-hgarg-1:namespace=namespaces x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D7f4e523d01f2cc3765ac8934da3d14db775ff872 x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 401 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) "POST /?cmd=unbundle HTTP/1.1" 200 - x-hgarg-1:heads=666f726365* (glob) - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-import.t mercurial-4.1~rc~trusty1/tests/test-import.t --- mercurial-4.0.2~trusty1/tests/test-import.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-import.t 2017-01-18 16:54:51.000000000 +0000 @@ -431,10 +431,10 @@ parent: 0 Test that "hg rollback" doesn't restore dirstate to one at the -beginning of the rollbacked transaction in not-"parent-gone" case. +beginning of the rolled back transaction in not-"parent-gone" case. invoking pretxncommit hook will cause marking '.hg/dirstate' as a file -to be restored at rollbacking, after DirstateTransactionPlan (see wiki +to be restored when rolling back, after DirstateTransactionPlan (see wiki page for detail). $ hg --cwd b branch -q foobar @@ -451,7 +451,7 @@ $ hg --cwd b update -q -C 0 $ hg --cwd b --config extensions.strip= strip -q 1 -Test visibility of in-memory distate changes inside transaction to +Test visibility of in-memory dirstate changes inside transaction to external process $ echo foo > a/foo @@ -1517,6 +1517,13 @@ | o initial [Babar] 2: +8/-0 +Adding those config options should not change the output of diffstat. Bugfix #4755. + + $ hg log -r . --template '{diffstat}\n' + 1: +1/-0 + $ hg log -r . --template '{diffstat}\n' --config diff.git=1 \ + > --config diff.noprefix=1 + 1: +1/-0 Importing with some success and some errors: diff -Nru mercurial-4.0.2~trusty1/tests/test-install.t mercurial-4.1~rc~trusty1/tests/test-install.t --- mercurial-4.0.2~trusty1/tests/test-install.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-install.t 2017-01-18 16:54:51.000000000 +0000 @@ -11,6 +11,9 @@ checking Mercurial custom build (*) (glob) checking module policy (*) (glob) checking installed modules (*mercurial)... (glob) + checking registered compression engines (*zlib*) (glob) + checking available compression engines (*zlib*) (glob) + checking available compression engines for wire protocol (*zlib*) (glob) checking templates (*mercurial?templates)... (glob) checking default template (*mercurial?templates?map-cmdline.default) (glob) checking commit editor... (* -c "import sys; sys.exit(0)") (glob) @@ -21,6 +24,9 @@ $ hg debuginstall -Tjson | sed 's|\\\\|\\|g' [ { + "compengines": ["bz2", "bz2truncated", "none", "zlib"*], (glob) + "compenginesavail": ["bz2", "bz2truncated", "none", "zlib"*], (glob) + "compenginesserver": [*"zlib"*], (glob) "defaulttemplate": "*mercurial?templates?map-cmdline.default", (glob) "defaulttemplateerror": null, "defaulttemplatenotfound": "default", @@ -58,6 +64,9 @@ checking Mercurial custom build (*) (glob) checking module policy (*) (glob) checking installed modules (*mercurial)... (glob) + checking registered compression engines (*zlib*) (glob) + checking available compression engines (*zlib*) (glob) + checking available compression engines for wire protocol (*zlib*) (glob) checking templates (*mercurial?templates)... (glob) checking default template (*mercurial?templates?map-cmdline.default) (glob) checking commit editor... (* -c "import sys; sys.exit(0)") (glob) @@ -85,6 +94,9 @@ checking Mercurial custom build (*) (glob) checking module policy (*) (glob) checking installed modules (*mercurial)... (glob) + checking registered compression engines (*zlib*) (glob) + checking available compression engines (*zlib*) (glob) + checking available compression engines for wire protocol (*zlib*) (glob) checking templates (*mercurial?templates)... (glob) checking default template (*mercurial?templates?map-cmdline.default) (glob) checking commit editor... (* -c "import sys; sys.exit(0)") (glob) diff -Nru mercurial-4.0.2~trusty1/tests/test-keyword.t mercurial-4.1~rc~trusty1/tests/test-keyword.t --- mercurial-4.0.2~trusty1/tests/test-keyword.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-keyword.t 2017-01-18 16:54:51.000000000 +0000 @@ -1117,7 +1117,7 @@ bar ======= foo - >>>>>>> merge rev: 85d2d2d732a5 - test: simplemerge + >>>>>>> merge rev: 85d2d2d732a5 - test: simplemerge resolve to local, m must contain hash of last change (local parent) diff -Nru mercurial-4.0.2~trusty1/tests/test-largefiles-wireproto.t mercurial-4.1~rc~trusty1/tests/test-largefiles-wireproto.t --- mercurial-4.0.2~trusty1/tests/test-largefiles-wireproto.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-largefiles-wireproto.t 2017-01-18 16:54:51.000000000 +0000 @@ -347,7 +347,7 @@ searching 2 changesets for largefiles verified existence of 2 revisions of 2 largefiles $ tail -1 access.log - 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a (glob) + 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3D972a1a11f19934401291cc99117ec614933374ce%3Bstatlfile+sha%3Dc801c9cfe94400963fcb683246217d5db77f9a9a x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) $ hg -R batchverifyclone update getting changed largefiles 2 largefiles updated, 0 removed @@ -384,11 +384,11 @@ searching 3 changesets for largefiles verified existence of 3 revisions of 3 largefiles $ tail -1 access.log - 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c (glob) + 127.0.0.1 - - [*] "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=statlfile+sha%3Dc8559c3c9cfb42131794b7d8009230403b9b454c x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) $ killdaemons.py -largefiles should not ask for password again after succesfull authorization +largefiles should not ask for password again after successful authorization $ hg init credentialmain $ cd credentialmain diff -Nru mercurial-4.0.2~trusty1/tests/test-linerange.py mercurial-4.1~rc~trusty1/tests/test-linerange.py --- mercurial-4.0.2~trusty1/tests/test-linerange.py 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-linerange.py 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,232 @@ +from __future__ import absolute_import + +import unittest +from mercurial import error, mdiff + +# for readability, line numbers are 0-origin +text1 = ''' + 00 at OLD + 01 at OLD + 02 at OLD +02 at NEW, 03 at OLD +03 at NEW, 04 at OLD +04 at NEW, 05 at OLD +05 at NEW, 06 at OLD + 07 at OLD + 08 at OLD + 09 at OLD + 10 at OLD + 11 at OLD +'''[1:] # strip initial LF + +text2 = ''' +00 at NEW +01 at NEW +02 at NEW, 03 at OLD +03 at NEW, 04 at OLD +04 at NEW, 05 at OLD +05 at NEW, 06 at OLD +06 at NEW +07 at NEW +08 at NEW +09 at NEW +10 at NEW +11 at NEW +'''[1:] # strip initial LF + +def filteredblocks(blocks, rangeb): + """return `rangea` extracted from `blocks` coming from + `mdiff.blocksinrange` along with the mask of blocks within rangeb. + """ + filtered, rangea = mdiff.blocksinrange(blocks, rangeb) + skipped = [b not in filtered for b in blocks] + return rangea, skipped + +class blocksinrangetests(unittest.TestCase): + + def setUp(self): + self.blocks = list(mdiff.allblocks(text1, text2)) + assert self.blocks == [ + ([0, 3, 0, 2], '!'), + ((3, 7, 2, 6), '='), + ([7, 12, 6, 12], '!'), + ((12, 12, 12, 12), '='), + ], self.blocks + + def testWithinEqual(self): + """linerange within an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^ + linerange2 = (3, 5) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (4, 6)) + self.assertEqual(skipped, [True, False, True, True]) + + def testWithinEqualStrictly(self): + """linerange matching exactly an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^^^ + linerange2 = (2, 6) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (3, 7)) + self.assertEqual(skipped, [True, False, True, True]) + + def testWithinEqualLowerbound(self): + """linerange at beginning of an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^ + linerange2 = (2, 4) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (3, 5)) + self.assertEqual(skipped, [True, False, True, True]) + + def testWithinEqualLowerboundOneline(self): + """oneline-linerange at beginning of an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^ + linerange2 = (2, 3) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (3, 4)) + self.assertEqual(skipped, [True, False, True, True]) + + def testWithinEqualUpperbound(self): + """linerange at end of an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^^ + linerange2 = (3, 6) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (4, 7)) + self.assertEqual(skipped, [True, False, True, True]) + + def testWithinEqualUpperboundOneLine(self): + """oneline-linerange at end of an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^ + linerange2 = (5, 6) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (6, 7)) + self.assertEqual(skipped, [True, False, True, True]) + + def testWithinFirstBlockNeq(self): + """linerange within the first "!" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^ + # | (empty) + # ^ + # ^^ + for linerange2 in [ + (0, 1), + (1, 1), + (1, 2), + (0, 2), + ]: + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (0, 3)) + self.assertEqual(skipped, [False, True, True, True]) + + def testWithinLastBlockNeq(self): + """linerange within the last "!" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^ + # ^ + # | (empty) + # ^^^^^^ + # ^ + for linerange2 in [ + (6, 7), + (7, 8), + (7, 7), + (6, 12), + (11, 12), + ]: + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (7, 12)) + self.assertEqual(skipped, [True, True, False, True]) + + def testAccrossTwoBlocks(self): + """linerange accross two blocks""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^^^ + linerange2 = (1, 5) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (0, 6)) + self.assertEqual(skipped, [False, False, True, True]) + + def testCrossingSeveralBlocks(self): + """linerange accross three blocks""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^^^^^^ + linerange2 = (1, 8) + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, (0, 12)) + self.assertEqual(skipped, [False, False, False, True]) + + def testStartInEqBlock(self): + """linerange starting in an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^^^ + # ^^^^^^^ + for linerange2, expectedlinerange1 in [ + ((5, 9), (6, 12)), + ((4, 11), (5, 12)), + ]: + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, expectedlinerange1) + self.assertEqual(skipped, [True, False, False, True]) + + def testEndInEqBlock(self): + """linerange ending in an "=" block""" + # IDX 0 1 + # 012345678901 + # SRC NNOOOONNNNNN (New/Old) + # ^^ + # ^^^^^ + for linerange2, expectedlinerange1 in [ + ((1, 3), (0, 4)), + ((0, 4), (0, 5)), + ]: + linerange1, skipped = filteredblocks(self.blocks, linerange2) + self.assertEqual(linerange1, expectedlinerange1) + self.assertEqual(skipped, [False, False, True, True]) + + def testOutOfRange(self): + """linerange exceeding file size""" + exctype = error.Abort + for linerange2 in [ + (0, 34), + (15, 12), + ]: + # Could be `with self.assertRaises(error.Abort)` but python2.6 + # does not have assertRaises context manager. + try: + mdiff.blocksinrange(self.blocks, linerange2) + except exctype as exc: + self.assertTrue('line range exceeds file size' in str(exc)) + else: + self.fail('%s not raised' % exctype.__name__) + +if __name__ == '__main__': + import silenttestrunner + silenttestrunner.main(__name__) diff -Nru mercurial-4.0.2~trusty1/tests/test-merge7.t mercurial-4.1~rc~trusty1/tests/test-merge7.t --- mercurial-4.0.2~trusty1/tests/test-merge7.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-merge7.t 2017-01-18 16:54:51.000000000 +0000 @@ -99,7 +99,7 @@ $ cat test.txt one - <<<<<<< working copy: 50c3a7e29886 - test: Merge 1 + <<<<<<< working copy: 50c3a7e29886 - test: Merge 1 two-point-five ======= two-point-one diff -Nru mercurial-4.0.2~trusty1/tests/test-merge-changedelete.t mercurial-4.1~rc~trusty1/tests/test-merge-changedelete.t --- mercurial-4.0.2~trusty1/tests/test-merge-changedelete.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-merge-changedelete.t 2017-01-18 16:54:51.000000000 +0000 @@ -107,7 +107,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Interactive merge: @@ -171,7 +171,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Interactive merge with bad input: @@ -247,7 +247,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Interactive merge with not enough input: @@ -310,7 +310,7 @@ changed2 ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Choose local versions of files @@ -635,7 +635,7 @@ ||||||| base ======= changed1 - >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, cha... + >>>>>>> merge rev: 10f9a0a634e8 - test: removed file1, changed file2, chan... Exercise transitions between local, other, fail and prompt, and make sure the dirstate stays consistent. (Compare with each other and to the above diff -Nru mercurial-4.0.2~trusty1/tests/test-merge-force.t mercurial-4.1~rc~trusty1/tests/test-merge-force.t --- mercurial-4.0.2~trusty1/tests/test-merge-force.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-merge-force.t 2017-01-18 16:54:51.000000000 +0000 @@ -379,7 +379,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content1_content4-untracked content2 @@ -409,7 +409,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content2_content4-untracked content2 @@ -439,7 +439,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content3_content3-untracked content2 @@ -451,7 +451,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_content3_content4-untracked content2 @@ -481,7 +481,7 @@ content1 ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M content1_content2_missing_content4-untracked content2 @@ -564,7 +564,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_content2_content4-untracked content2 @@ -587,7 +587,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_content3_content3-untracked content2 @@ -598,7 +598,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_content3_content4-untracked content2 @@ -621,7 +621,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_missing_content4-untracked <<<<<<< working copy: 0447570f1af6 - test: local @@ -629,7 +629,7 @@ ||||||| base ======= content2 - >>>>>>> merge rev: 85100b8c675b - test: remote + >>>>>>> merge rev: 85100b8c675b - test: remote M missing_content2_missing_missing-tracked content2 diff -Nru mercurial-4.0.2~trusty1/tests/test-merge-revert2.t mercurial-4.1~rc~trusty1/tests/test-merge-revert2.t --- mercurial-4.0.2~trusty1/tests/test-merge-revert2.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-merge-revert2.t 2017-01-18 16:54:51.000000000 +0000 @@ -56,7 +56,7 @@ @@ -1,3 +1,7 @@ added file1 another line of text - +<<<<<<< working copy: c3fa057dd86f - test: added file1 and file2 + +<<<<<<< working copy: c3fa057dd86f - test: added file1 and file2 +changed file1 different +======= changed file1 diff -Nru mercurial-4.0.2~trusty1/tests/test-merge-tools.t mercurial-4.1~rc~trusty1/tests/test-merge-tools.t --- mercurial-4.0.2~trusty1/tests/test-merge-tools.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-merge-tools.t 2017-01-18 16:54:51.000000000 +0000 @@ -75,11 +75,11 @@ [1] $ aftermerge # cat f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 ======= revision 2 - >>>>>>> merge rev: 0185f4e0cf02 - test: revision 2 + >>>>>>> merge rev: 0185f4e0cf02 - test: revision 2 space # hg stat M f @@ -935,7 +935,7 @@ # hg update -C 1 $ hg merge -r 4 --config merge-tools.true.premerge=keep merging f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ======= @@ -948,7 +948,7 @@ (branch merge, don't forget to commit) $ aftermerge # cat f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ======= @@ -969,7 +969,7 @@ # hg update -C 1 $ hg merge -r 4 --config merge-tools.true.premerge=keep-merge3 merging f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ||||||| base @@ -985,7 +985,7 @@ (branch merge, don't forget to commit) $ aftermerge # cat f - <<<<<<< working copy: ef83787e2614 - test: revision 1 + <<<<<<< working copy: ef83787e2614 - test: revision 1 revision 1 space ||||||| base @@ -1209,3 +1209,15 @@ [1] #endif + +Verify naming of temporary files and that extension is preserved: + + $ hg update -q -C 1 + $ hg mv f f.txt + $ hg ci -qm "f.txt" + $ hg update -q -C 2 + $ hg merge -y -r tip --tool echo --config merge-tools.echo.args='$base $local $other $output' + merging f and f.txt to f.txt + */f~base.?????? $TESTTMP/f.txt.orig */f~other.??????.txt $TESTTMP/f.txt (glob) + 0 files updated, 1 files merged, 0 files removed, 0 files unresolved + (branch merge, don't forget to commit) diff -Nru mercurial-4.0.2~trusty1/tests/test-merge-types.t mercurial-4.1~rc~trusty1/tests/test-merge-types.t --- mercurial-4.0.2~trusty1/tests/test-merge-types.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-merge-types.t 2017-01-18 16:54:51.000000000 +0000 @@ -367,14 +367,14 @@ 2 ======= 1 - >>>>>>> merge rev: 2e60aa20b912 - test: 1 + >>>>>>> merge rev: 2e60aa20b912 - test: 1 $ tellmeabout b b is a plain file with content: <<<<<<< working copy: 0c617753b41b - test: 2 2 ======= 1 - >>>>>>> merge rev: 2e60aa20b912 - test: 1 + >>>>>>> merge rev: 2e60aa20b912 - test: 1 $ tellmeabout c c is a plain file with content: x @@ -418,14 +418,14 @@ [1] $ tellmeabout a a is a plain file with content: - <<<<<<< working copy: 2e60aa20b912 - test: 1 + <<<<<<< working copy: 2e60aa20b912 - test: 1 1 ======= 2 >>>>>>> merge rev: 0c617753b41b - test: 2 $ tellmeabout b b is an executable file with content: - <<<<<<< working copy: 2e60aa20b912 - test: 1 + <<<<<<< working copy: 2e60aa20b912 - test: 1 1 ======= 2 diff -Nru mercurial-4.0.2~trusty1/tests/test-mq-qrefresh-replace-log-message.t mercurial-4.1~rc~trusty1/tests/test-mq-qrefresh-replace-log-message.t --- mercurial-4.0.2~trusty1/tests/test-mq-qrefresh-replace-log-message.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-mq-qrefresh-replace-log-message.t 2017-01-18 16:54:51.000000000 +0000 @@ -199,7 +199,7 @@ test saving last-message.txt -Test visibility of in-memory distate changes outside transaction to +Test visibility of in-memory dirstate changes outside transaction to external process $ cat > $TESTTMP/checkvisibility.sh < # public changeset are not obsolete > publish=false > [ui] - > logtemplate="{rev}:{node|short} ({phase}) [{tags} {bookmarks}] {desc|firstline}\n" + > logtemplate="{rev}:{node|short} ({phase}{if(troubles, ' {troubles}')}) [{tags} {bookmarks}] {desc|firstline}\n" > EOF $ mkcommit() { > echo "$1" > "$1" @@ -203,7 +203,7 @@ $ hg --hidden phase --public 2 $ hg log -G - @ 5:5601fb93a350 (draft) [tip ] add new_3_c + @ 5:5601fb93a350 (draft bumped) [tip ] add new_3_c | | o 2:245bde4270cd (public) [ ] add original_c |/ @@ -220,7 +220,7 @@ the public changeset $ hg log --hidden -r 'bumped()' - 5:5601fb93a350 (draft) [tip ] add new_3_c + 5:5601fb93a350 (draft bumped) [tip ] add new_3_c And that we can't push bumped changeset @@ -242,7 +242,7 @@ We need to create a clone of 5 and add a special marker with a flag $ hg summary - parent: 5:5601fb93a350 tip + parent: 5:5601fb93a350 tip (bumped) add new_3_c branch: default commit: (clean) @@ -477,7 +477,7 @@ $ hg log -r 'obsolete()' 4:94b33453f93b (draft) [ ] add original_d $ hg summary - parent: 5:cda648ca50f5 tip + parent: 5:cda648ca50f5 tip (unstable) add original_e branch: default commit: (clean) @@ -485,7 +485,7 @@ phases: 3 draft unstable: 1 changesets $ hg log -G -r '::unstable()' - @ 5:cda648ca50f5 (draft) [tip ] add original_e + @ 5:cda648ca50f5 (draft unstable) [tip ] add original_e | x 4:94b33453f93b (draft) [ ] add original_d | @@ -527,7 +527,7 @@ 2:245bde4270cd (public) [ ] add original_c 3:6f9641995072 (draft) [ ] add n3w_3_c 4:94b33453f93b (draft) [ ] add original_d - 5:cda648ca50f5 (draft) [tip ] add original_e + 5:cda648ca50f5 (draft unstable) [tip ] add original_e $ hg push ../tmpf -f # -f because be push unstable too pushing to ../tmpf searching for changes @@ -548,7 +548,7 @@ Do not warn about new head when the new head is a successors of a remote one $ hg log -G - @ 5:cda648ca50f5 (draft) [tip ] add original_e + @ 5:cda648ca50f5 (draft unstable) [tip ] add original_e | x 4:94b33453f93b (draft) [ ] add original_d | @@ -719,8 +719,6 @@ $ hg debugobsolete -r6 -T '{flag} {get(metadata, "user")}\n' 0 test -#if serve - Test the debug output for exchange ---------------------------------- @@ -746,6 +744,8 @@ $ hg up tip 2 files updated, 0 files merged, 0 files removed, 0 files unresolved +#if serve + $ hg serve -n test -p $HGPORT -d --pid-file=hg.pid -A access.log -E errors.log $ cat hg.pid >> $DAEMON_PIDS @@ -796,8 +796,50 @@ $ echo '[experimental]' >> $HGRCPATH $ echo "evolution=createmarkers,exchange" >> $HGRCPATH + $ rm hg.pid access.log errors.log #endif +Several troubles on the same changeset (create an unstable and bumped changeset) + + $ hg debugobsolete `getid obsolete_e` + $ hg debugobsolete `getid original_c` `getid babar` + $ hg log --config ui.logtemplate= -r 'bumped() and unstable()' + changeset: 7:50c51b361e60 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + trouble: unstable, bumped + summary: add babar + + +test the "troubles" templatekw + + $ hg log -r 'bumped() and unstable()' + 7:50c51b361e60 (draft unstable bumped) [ ] add babar + +test the default cmdline template + + $ hg log -T default -r 'bumped()' + changeset: 7:50c51b361e60 + user: test + date: Thu Jan 01 00:00:00 1970 +0000 + trouble: unstable, bumped + summary: add babar + + +test summary output + + $ hg up -r 'bumped() and unstable()' + 1 files updated, 0 files merged, 1 files removed, 0 files unresolved + $ hg summary + parent: 7:50c51b361e60 (unstable, bumped) + add babar + branch: default + commit: (clean) + update: 2 new changesets (update) + phases: 4 draft + unstable: 2 changesets + bumped: 1 changesets + Test incoming/outcoming with changesets obsoleted remotely, known locally =============================================================================== @@ -1211,7 +1253,7 @@ 2 1ab51af8f9b41ef8c7f6f3312d4706d870b1fb74 29346082e4a9e27042b62d2da0e2de211c027621 0 \(.*\) {'user': 'test'} (re) 3 4715cf767440ed891755448016c2b8cf70760c30 7ae79c5d60f049c7b0dd02f5f25b9d60aaf7b36d 0 \(.*\) {'user': 'test'} (re) $ hg debugobsolete --delete 1 --delete 3 - deleted 2 obsolescense markers + deleted 2 obsolescence markers $ hg debugobsolete cb9a9f314b8b07ba71012fcdbc544b5a4d82ff5b f9bd49731b0b175e42992a3c8fa6c678b2bc11f1 0 \(.*\) {'user': 'test'} (re) 1ab51af8f9b41ef8c7f6f3312d4706d870b1fb74 29346082e4a9e27042b62d2da0e2de211c027621 0 \(.*\) {'user': 'test'} (re) diff -Nru mercurial-4.0.2~trusty1/tests/test-profile.t mercurial-4.1~rc~trusty1/tests/test-profile.t --- mercurial-4.0.2~trusty1/tests/test-profile.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-profile.t 2017-01-18 16:54:51.000000000 +0000 @@ -8,25 +8,27 @@ test --profile - $ hg --profile st 2>../out + $ prof='hg --config profiling.type=ls --profile' + + $ $prof st 2>../out $ grep CallCount ../out > /dev/null || cat ../out - $ hg --profile --config profiling.output=../out st + $ $prof --config profiling.output=../out st $ grep CallCount ../out > /dev/null || cat ../out - $ hg --profile --config profiling.output=blackbox --config extensions.blackbox= st + $ $prof --config profiling.output=blackbox --config extensions.blackbox= st $ grep CallCount .hg/blackbox.log > /dev/null || cat .hg/blackbox.log - $ hg --profile --config profiling.format=text st 2>../out + $ $prof --config profiling.format=text st 2>../out $ grep CallCount ../out > /dev/null || cat ../out $ echo "[profiling]" >> $HGRCPATH $ echo "format=kcachegrind" >> $HGRCPATH - $ hg --profile st 2>../out + $ $prof st 2>../out $ grep 'events: Ticks' ../out > /dev/null || cat ../out - $ hg --profile --config profiling.output=../out st + $ $prof --config profiling.output=../out st $ grep 'events: Ticks' ../out > /dev/null || cat ../out #endif @@ -35,7 +37,7 @@ Profiling of HTTP requests works - $ hg --profile --config profiling.format=text --config profiling.output=../profile.log serve -d -p $HGPORT --pid-file ../hg.pid -A ../access.log + $ $prof --config profiling.format=text --config profiling.output=../profile.log serve -d -p $HGPORT --pid-file ../hg.pid -A ../access.log $ cat ../hg.pid >> $DAEMON_PIDS $ hg -q clone -U http://localhost:$HGPORT ../clone @@ -45,4 +47,49 @@ #endif +Install an extension that can sleep and guarantee a profiler has time to run + + $ cat >> sleepext.py << EOF + > import time + > from mercurial import cmdutil, commands + > cmdtable = {} + > command = cmdutil.command(cmdtable) + > @command('sleep', [], 'hg sleep') + > def sleep(ui, *args, **kwargs): + > time.sleep(0.1) + > EOF + + $ cat >> $HGRCPATH << EOF + > [extensions] + > sleep = `pwd`/sleepext.py + > EOF + +statistical profiler works + + $ hg --profile sleep 2>../out + $ grep Sample ../out + Sample count: \d+ (re) + +Various statprof formatters work + + $ hg --profile --config profiling.statformat=byline sleep 2>../out + $ head -n 1 ../out + % cumulative self + $ grep Sample ../out + Sample count: \d+ (re) + + $ hg --profile --config profiling.statformat=bymethod sleep 2>../out + $ head -n 1 ../out + % cumulative self + $ grep Sample ../out + Sample count: \d+ (re) + + $ hg --profile --config profiling.statformat=hotpath sleep 2>../out + $ grep Sample ../out + Sample count: \d+ (re) + + $ hg --profile --config profiling.statformat=json sleep 2>../out + $ cat ../out + \[\[\d+.* (re) + $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-propertycache.py mercurial-4.1~rc~trusty1/tests/test-propertycache.py --- mercurial-4.0.2~trusty1/tests/test-propertycache.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-propertycache.py 2017-01-18 16:54:51.000000000 +0000 @@ -46,7 +46,7 @@ # these tests on the real object to detect regression. repopath = os.path.join(os.environ['TESTTMP'], 'repo') assert subprocess.call(['hg', 'init', repopath]) == 0 -ui = uimod.ui() +ui = uimod.ui.load() repo = hg.repository(ui, path=repopath).unfiltered() diff -Nru mercurial-4.0.2~trusty1/tests/test-pull-update.t mercurial-4.1~rc~trusty1/tests/test-pull-update.t --- mercurial-4.0.2~trusty1/tests/test-pull-update.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-pull-update.t 2017-01-18 16:54:51.000000000 +0000 @@ -130,7 +130,7 @@ Test that updating deactivates current active bookmark, if the destination of the update is explicitly specified, and it doesn't -match with the name of any exsiting bookmarks. +match with the name of any existing bookmarks. $ cd ../t $ hg bookmark -d active-after-pull diff -Nru mercurial-4.0.2~trusty1/tests/test-push-hook-lock.t mercurial-4.1~rc~trusty1/tests/test-push-hook-lock.t --- mercurial-4.0.2~trusty1/tests/test-push-hook-lock.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-push-hook-lock.t 1970-01-01 00:00:00.000000000 +0000 @@ -1,50 +0,0 @@ - $ hg init 1 - - $ echo '[ui]' >> 1/.hg/hgrc - $ echo 'timeout = 10' >> 1/.hg/hgrc - - $ echo foo > 1/foo - $ hg --cwd 1 ci -A -m foo - adding foo - - $ hg clone 1 2 - updating to branch default - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - - $ hg clone 2 3 - updating to branch default - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - - $ cat < $TESTTMP/debuglocks-pretxn-hook.sh - > hg debuglocks - > true - > EOF - $ echo '[hooks]' >> 2/.hg/hgrc - $ echo "pretxnchangegroup.a = sh $TESTTMP/debuglocks-pretxn-hook.sh" >> 2/.hg/hgrc - $ echo 'changegroup.push = hg push -qf ../1' >> 2/.hg/hgrc - - $ echo bar >> 3/foo - $ hg --cwd 3 ci -m bar - - $ hg --cwd 3 push ../2 --config devel.legacy.exchange=bundle1 - pushing to ../2 - searching for changes - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - lock: user *, process * (*s) (glob) - wlock: free - - $ hg --cwd 1 --config extensions.strip= strip tip -q - $ hg --cwd 2 --config extensions.strip= strip tip -q - $ hg --cwd 3 push ../2 # bundle2+ - pushing to ../2 - searching for changes - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - lock: user *, process * (*s) (glob) - wlock: user *, process * (*s) (glob) - diff -Nru mercurial-4.0.2~trusty1/tests/test-push-r.t mercurial-4.1~rc~trusty1/tests/test-push-r.t --- mercurial-4.0.2~trusty1/tests/test-push-r.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-push-r.t 1970-01-01 00:00:00.000000000 +0000 @@ -1,149 +0,0 @@ - $ hg init test - $ cd test - $ hg unbundle "$TESTDIR/bundles/remote.hg" - adding changesets - adding manifests - adding file changes - added 9 changesets with 7 changes to 4 files (+1 heads) - (run 'hg heads' to see heads, 'hg merge' to merge) - $ hg up tip - 1 files updated, 0 files merged, 0 files removed, 0 files unresolved - $ cd .. - - $ for i in 0 1 2 3 4 5 6 7 8; do - > echo - > mkdir test-"$i" - > hg --cwd test-"$i" init - > hg -R test push -r "$i" test-"$i" - > cd test-"$i" - > hg verify - > cd .. - > done - - pushing to test-0 - searching for changes - adding changesets - adding manifests - adding file changes - added 1 changesets with 1 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 1 changesets, 1 total revisions - - pushing to test-1 - searching for changes - adding changesets - adding manifests - adding file changes - added 2 changesets with 2 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 2 changesets, 2 total revisions - - pushing to test-2 - searching for changes - adding changesets - adding manifests - adding file changes - added 3 changesets with 3 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 3 changesets, 3 total revisions - - pushing to test-3 - searching for changes - adding changesets - adding manifests - adding file changes - added 4 changesets with 4 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 4 changesets, 4 total revisions - - pushing to test-4 - searching for changes - adding changesets - adding manifests - adding file changes - added 2 changesets with 2 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 2 changesets, 2 total revisions - - pushing to test-5 - searching for changes - adding changesets - adding manifests - adding file changes - added 3 changesets with 3 changes to 1 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 1 files, 3 changesets, 3 total revisions - - pushing to test-6 - searching for changes - adding changesets - adding manifests - adding file changes - added 4 changesets with 5 changes to 2 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 2 files, 4 changesets, 5 total revisions - - pushing to test-7 - searching for changes - adding changesets - adding manifests - adding file changes - added 5 changesets with 6 changes to 3 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 3 files, 5 changesets, 6 total revisions - - pushing to test-8 - searching for changes - adding changesets - adding manifests - adding file changes - added 5 changesets with 5 changes to 2 files - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 2 files, 5 changesets, 5 total revisions - - $ cd test-8 - - $ hg pull ../test-7 - pulling from ../test-7 - searching for changes - adding changesets - adding manifests - adding file changes - added 4 changesets with 2 changes to 3 files (+1 heads) - (run 'hg heads' to see heads, 'hg merge' to merge) - - $ hg verify - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - 4 files, 9 changesets, 7 total revisions - - $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-push.t mercurial-4.1~rc~trusty1/tests/test-push.t --- mercurial-4.0.2~trusty1/tests/test-push.t 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-push.t 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,299 @@ +================================== +Basic testing for the push command +================================== + +Testing of the '--rev' flag +=========================== + + $ hg init test-revflag + $ hg -R test-revflag unbundle "$TESTDIR/bundles/remote.hg" + adding changesets + adding manifests + adding file changes + added 9 changesets with 7 changes to 4 files (+1 heads) + (run 'hg heads' to see heads, 'hg merge' to merge) + + $ for i in 0 1 2 3 4 5 6 7 8; do + > echo + > hg init test-revflag-"$i" + > hg -R test-revflag push -r "$i" test-revflag-"$i" + > hg -R test-revflag-"$i" verify + > done + + pushing to test-revflag-0 + searching for changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 1 changesets, 1 total revisions + + pushing to test-revflag-1 + searching for changes + adding changesets + adding manifests + adding file changes + added 2 changesets with 2 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 2 changesets, 2 total revisions + + pushing to test-revflag-2 + searching for changes + adding changesets + adding manifests + adding file changes + added 3 changesets with 3 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 3 changesets, 3 total revisions + + pushing to test-revflag-3 + searching for changes + adding changesets + adding manifests + adding file changes + added 4 changesets with 4 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 4 changesets, 4 total revisions + + pushing to test-revflag-4 + searching for changes + adding changesets + adding manifests + adding file changes + added 2 changesets with 2 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 2 changesets, 2 total revisions + + pushing to test-revflag-5 + searching for changes + adding changesets + adding manifests + adding file changes + added 3 changesets with 3 changes to 1 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 1 files, 3 changesets, 3 total revisions + + pushing to test-revflag-6 + searching for changes + adding changesets + adding manifests + adding file changes + added 4 changesets with 5 changes to 2 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 2 files, 4 changesets, 5 total revisions + + pushing to test-revflag-7 + searching for changes + adding changesets + adding manifests + adding file changes + added 5 changesets with 6 changes to 3 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 3 files, 5 changesets, 6 total revisions + + pushing to test-revflag-8 + searching for changes + adding changesets + adding manifests + adding file changes + added 5 changesets with 5 changes to 2 files + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 2 files, 5 changesets, 5 total revisions + + $ cd test-revflag-8 + + $ hg pull ../test-revflag-7 + pulling from ../test-revflag-7 + searching for changes + adding changesets + adding manifests + adding file changes + added 4 changesets with 2 changes to 3 files (+1 heads) + (run 'hg heads' to see heads, 'hg merge' to merge) + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 4 files, 9 changesets, 7 total revisions + + $ cd .. + +Test server side validation during push +======================================= + + $ hg init test-validation + $ cd test-validation + + $ cat > .hg/hgrc < [server] + > validate=1 + > EOF + + $ echo alpha > alpha + $ echo beta > beta + $ hg addr + adding alpha + adding beta + $ hg ci -m 1 + + $ cd .. + $ hg clone test-validation test-validation-clone + updating to branch default + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved + +Test spurious filelog entries: + + $ cd test-validation-clone + $ echo blah >> beta + $ cp .hg/store/data/beta.i tmp1 + $ hg ci -m 2 + $ cp .hg/store/data/beta.i tmp2 + $ hg -q rollback + $ mv tmp2 .hg/store/data/beta.i + $ echo blah >> beta + $ hg ci -m '2 (corrupt)' + +Expected to fail: + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + beta@1: dddc47b3ba30 not in manifests + 2 files, 2 changesets, 4 total revisions + 1 integrity errors encountered! + (first damaged changeset appears to be 1) + [1] + + $ hg push + pushing to $TESTTMP/test-validation (glob) + searching for changes + adding changesets + adding manifests + adding file changes + transaction abort! + rollback completed + abort: received spurious file revlog entry + [255] + + $ hg -q rollback + $ mv tmp1 .hg/store/data/beta.i + $ echo beta > beta + +Test missing filelog entries: + + $ cp .hg/store/data/beta.i tmp + $ echo blah >> beta + $ hg ci -m '2 (corrupt)' + $ mv tmp .hg/store/data/beta.i + +Expected to fail: + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + beta@1: manifest refers to unknown revision dddc47b3ba30 + 2 files, 2 changesets, 2 total revisions + 1 integrity errors encountered! + (first damaged changeset appears to be 1) + [1] + + $ hg push + pushing to $TESTTMP/test-validation (glob) + searching for changes + adding changesets + adding manifests + adding file changes + transaction abort! + rollback completed + abort: missing file data for beta:dddc47b3ba30e54484720ce0f4f768a0f4b6efb9 - run hg verify + [255] + + $ cd .. + +Test push hook locking +===================== + + $ hg init 1 + + $ echo '[ui]' >> 1/.hg/hgrc + $ echo 'timeout = 10' >> 1/.hg/hgrc + + $ echo foo > 1/foo + $ hg --cwd 1 ci -A -m foo + adding foo + + $ hg clone 1 2 + updating to branch default + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + + $ hg clone 2 3 + updating to branch default + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + + $ cat < $TESTTMP/debuglocks-pretxn-hook.sh + > hg debuglocks + > true + > EOF + $ echo '[hooks]' >> 2/.hg/hgrc + $ echo "pretxnchangegroup.a = sh $TESTTMP/debuglocks-pretxn-hook.sh" >> 2/.hg/hgrc + $ echo 'changegroup.push = hg push -qf ../1' >> 2/.hg/hgrc + + $ echo bar >> 3/foo + $ hg --cwd 3 ci -m bar + + $ hg --cwd 3 push ../2 --config devel.legacy.exchange=bundle1 + pushing to ../2 + searching for changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + lock: user *, process * (*s) (glob) + wlock: free + + $ hg --cwd 1 --config extensions.strip= strip tip -q + $ hg --cwd 2 --config extensions.strip= strip tip -q + $ hg --cwd 3 push ../2 # bundle2+ + pushing to ../2 + searching for changes + adding changesets + adding manifests + adding file changes + added 1 changesets with 1 changes to 1 files + lock: user *, process * (*s) (glob) + wlock: user *, process * (*s) (glob) + diff -Nru mercurial-4.0.2~trusty1/tests/test-push-validation.t mercurial-4.1~rc~trusty1/tests/test-push-validation.t --- mercurial-4.0.2~trusty1/tests/test-push-validation.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-push-validation.t 1970-01-01 00:00:00.000000000 +0000 @@ -1,92 +0,0 @@ - $ hg init test - $ cd test - - $ cat > .hg/hgrc < [server] - > validate=1 - > EOF - - $ echo alpha > alpha - $ echo beta > beta - $ hg addr - adding alpha - adding beta - $ hg ci -m 1 - - $ cd .. - $ hg clone test test-clone - updating to branch default - 2 files updated, 0 files merged, 0 files removed, 0 files unresolved - -Test spurious filelog entries: - - $ cd test-clone - $ echo blah >> beta - $ cp .hg/store/data/beta.i tmp1 - $ hg ci -m 2 - $ cp .hg/store/data/beta.i tmp2 - $ hg -q rollback - $ mv tmp2 .hg/store/data/beta.i - $ echo blah >> beta - $ hg ci -m '2 (corrupt)' - -Expected to fail: - - $ hg verify - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - beta@1: dddc47b3ba30 not in manifests - 2 files, 2 changesets, 4 total revisions - 1 integrity errors encountered! - (first damaged changeset appears to be 1) - [1] - - $ hg push - pushing to $TESTTMP/test (glob) - searching for changes - adding changesets - adding manifests - adding file changes - transaction abort! - rollback completed - abort: received spurious file revlog entry - [255] - - $ hg -q rollback - $ mv tmp1 .hg/store/data/beta.i - $ echo beta > beta - -Test missing filelog entries: - - $ cp .hg/store/data/beta.i tmp - $ echo blah >> beta - $ hg ci -m '2 (corrupt)' - $ mv tmp .hg/store/data/beta.i - -Expected to fail: - - $ hg verify - checking changesets - checking manifests - crosschecking files in changesets and manifests - checking files - beta@1: manifest refers to unknown revision dddc47b3ba30 - 2 files, 2 changesets, 2 total revisions - 1 integrity errors encountered! - (first damaged changeset appears to be 1) - [1] - - $ hg push - pushing to $TESTTMP/test (glob) - searching for changes - adding changesets - adding manifests - adding file changes - transaction abort! - rollback completed - abort: missing file data for beta:dddc47b3ba30e54484720ce0f4f768a0f4b6efb9 - run hg verify - [255] - - $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-qrecord.t mercurial-4.1~rc~trusty1/tests/test-qrecord.t --- mercurial-4.0.2~trusty1/tests/test-qrecord.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-qrecord.t 2017-01-18 16:54:51.000000000 +0000 @@ -9,6 +9,9 @@ record extension - commands to interactively select changes for commit/qrefresh (DEPRECATED) + The feature provided by this extension has been moved into core Mercurial as + 'hg commit --interactive'. + (use 'hg help extensions' for information on enabling extensions) help qrecord (no record) diff -Nru mercurial-4.0.2~trusty1/tests/test-rebase-base.t mercurial-4.1~rc~trusty1/tests/test-rebase-base.t --- mercurial-4.0.2~trusty1/tests/test-rebase-base.t 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rebase-base.t 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,393 @@ + $ cat >> $HGRCPATH < [extensions] + > rebase= + > drawdag=$TESTDIR/drawdag.py + > + > [phases] + > publish=False + > + > [alias] + > tglog = log -G --template "{rev}: {desc}" + > EOF + + $ rebasewithdag() { + > N=`$PYTHON -c "print($N+1)"` + > hg init repo$N && cd repo$N + > hg debugdrawdag + > hg rebase "$@" > _rebasetmp + > r=$? + > grep -v 'saved backup bundle' _rebasetmp + > [ $r -eq 0 ] && hg tglog + > cd .. + > return $r + > } + +Single branching point, without merge: + + $ rebasewithdag -b D -d Z <<'EOS' + > D E + > |/ + > Z B C # C: branching point, E should be picked + > \|/ # B should not be picked + > A + > | + > R + > EOS + rebasing 3:d6003a550c2c "C" (C) + rebasing 5:4526cf523425 "D" (D) + rebasing 6:b296604d9846 "E" (E tip) + o 6: E + | + | o 5: D + |/ + o 4: C + | + o 3: Z + | + | o 2: B + |/ + o 1: A + | + o 0: R + +Multiple branching points caused by selecting a single merge changeset: + + $ rebasewithdag -b E -d Z <<'EOS' + > E + > /| + > B C D # B, C: multiple branching points + > | |/ # D should not be picked + > Z | / + > \|/ + > A + > | + > R + > EOS + rebasing 2:c1e6b162678d "B" (B) + rebasing 3:d6003a550c2c "C" (C) + rebasing 6:5251e0cb7302 "E" (E tip) + o 6: E + |\ + | o 5: C + | | + o | 4: B + |/ + o 3: Z + | + | o 2: D + |/ + o 1: A + | + o 0: R + +Rebase should not extend the "--base" revset using "descendants": + + $ rebasewithdag -b B -d Z <<'EOS' + > E + > /| + > Z B C # descendants(B) = B+E. With E, C will be included incorrectly + > \|/ + > A + > | + > R + > EOS + rebasing 2:c1e6b162678d "B" (B) + rebasing 5:5251e0cb7302 "E" (E tip) + o 5: E + |\ + | o 4: B + | | + | o 3: Z + | | + o | 2: C + |/ + o 1: A + | + o 0: R + +Rebase should not simplify the "--base" revset using "roots": + + $ rebasewithdag -b B+E -d Z <<'EOS' + > E + > /| + > Z B C # roots(B+E) = B. Without E, C will be missed incorrectly + > \|/ + > A + > | + > R + > EOS + rebasing 2:c1e6b162678d "B" (B) + rebasing 3:d6003a550c2c "C" (C) + rebasing 5:5251e0cb7302 "E" (E tip) + o 5: E + |\ + | o 4: C + | | + o | 3: B + |/ + o 2: Z + | + o 1: A + | + o 0: R + +The destination is one of the two branching points of a merge: + + $ rebasewithdag -b F -d Z <<'EOS' + > F + > / \ + > E D + > / / + > Z C + > \ / + > B + > | + > A + > EOS + nothing to rebase + [1] + +Multiple branching points caused by multiple bases (issue5420): + + $ rebasewithdag -b E1+E2+C2+B1 -d Z <<'EOS' + > Z E2 + > | / + > F E1 C2 + > |/ / + > E C1 B2 + > |/ / + > C B1 + > |/ + > B + > | + > A + > | + > R + > EOS + rebasing 3:a113dbaa660a "B1" (B1) + rebasing 5:06ce7b1cc8c2 "B2" (B2) + rebasing 6:0ac98cce32d3 "C1" (C1) + rebasing 8:781512f5e33d "C2" (C2) + rebasing 9:428d8c18f641 "E1" (E1) + rebasing 11:e1bf82f6b6df "E2" (E2) + o 12: E2 + | + o 11: E1 + | + | o 10: C2 + | | + | o 9: C1 + |/ + | o 8: B2 + | | + | o 7: B1 + |/ + o 6: Z + | + o 5: F + | + o 4: E + | + o 3: C + | + o 2: B + | + o 1: A + | + o 0: R + +Multiple branching points with multiple merges: + + $ rebasewithdag -b G+P -d Z <<'EOS' + > G H P + > |\ /| |\ + > F E D M N + > \|/| /| |\ + > Z C B I J K L + > \|/ |/ |/ + > A A A + > EOS + rebasing 2:dc0947a82db8 "C" (C) + rebasing 8:215e7b0814e1 "D" (D) + rebasing 9:03ca77807e91 "E" (E) + rebasing 10:afc707c82df0 "F" (F) + rebasing 13:018caa673317 "G" (G) + rebasing 14:4f710fbd68cb "H" (H) + rebasing 3:08ebfeb61bac "I" (I) + rebasing 4:a0a5005cec67 "J" (J) + rebasing 5:83780307a7e8 "K" (K) + rebasing 6:e131637a1cb6 "L" (L) + rebasing 11:d6fe3d11d95d "M" (M) + rebasing 12:fa1e02269063 "N" (N) + rebasing 15:448b1a498430 "P" (P tip) + o 15: P + |\ + | o 14: N + | |\ + o \ \ 13: M + |\ \ \ + | | | o 12: L + | | | | + | | o | 11: K + | | |/ + | o / 10: J + | |/ + o / 9: I + |/ + | o 8: H + | |\ + | | | o 7: G + | | |/| + | | | o 6: F + | | | | + | | o | 5: E + | | |/ + | o | 4: D + | |\| + +---o 3: C + | | + o | 2: Z + | | + | o 1: B + |/ + o 0: A + +Slightly more complex merge case (mentioned in https://www.mercurial-scm.org/pipermail/mercurial-devel/2016-November/091074.html): + + $ rebasewithdag -b A3+B3 -d Z <<'EOF' + > Z C1 A3 B3 + > | / / \ / \ + > M3 C0 A1 A2 B1 B2 + > | / | | | | + > M2 M1 C1 C1 M3 + > | + > M1 + > | + > M0 + > EOF + rebasing 4:8817fae53c94 "C0" (C0) + rebasing 6:06ca5dfe3b5b "B2" (B2) + rebasing 7:73508237b032 "C1" (C1) + rebasing 9:fdb955e2faed "A2" (A2) + rebasing 11:1b2f368c3cb5 "A3" (A3) + rebasing 10:0a33b0519128 "B1" (B1) + rebasing 12:bd6a37b5b67a "B3" (B3 tip) + o 12: B3 + |\ + | o 11: B1 + | | + | | o 10: A3 + | | |\ + | +---o 9: A2 + | | | + | o | 8: C1 + | | | + o | | 7: B2 + | | | + | o | 6: C0 + |/ / + o | 5: Z + | | + o | 4: M3 + | | + o | 3: M2 + | | + | o 2: A1 + |/ + o 1: M1 + | + o 0: M0 + +Mixed rebasable and non-rebasable bases (unresolved, issue5422): + + $ rebasewithdag -b C+D -d B <<'EOS' + > D + > / + > B C + > |/ + > A + > EOS + nothing to rebase + [1] + +Disconnected graph: + + $ rebasewithdag -b B -d Z <<'EOS' + > B + > | + > Z A + > EOS + nothing to rebase from 112478962961 to 48b9aae0607f + [1] + +Multiple roots. Roots are ancestors of dest: + + $ rebasewithdag -b B+D -d Z <<'EOF' + > D Z B + > \|\| + > C A + > EOF + rebasing 2:112478962961 "B" (B) + rebasing 3:b70f76719894 "D" (D) + o 4: D + | + | o 3: B + |/ + o 2: Z + |\ + | o 1: C + | + o 0: A + +Multiple roots. One root is not an ancestor of dest: + + $ rebasewithdag -b B+D -d Z <<'EOF' + > Z B D + > \|\| + > A C + > EOF + nothing to rebase from 86d01f49c0d9+b70f76719894 to 262e37e34f63 + [1] + +Multiple roots. One root is not an ancestor of dest. Select using a merge: + + $ rebasewithdag -b E -d Z <<'EOF' + > E + > |\ + > Z B D + > \|\| + > A C + > EOF + rebasing 2:86d01f49c0d9 "B" (B) + rebasing 5:539a0ff83ea9 "E" (E tip) + o 5: E + |\ + | o 4: B + | |\ + | | o 3: Z + | | | + o | | 2: D + |/ / + o / 1: C + / + o 0: A + +Multiple roots. Two children share two parents while dest has only one parent: + + $ rebasewithdag -b B+D -d Z <<'EOF' + > Z B D + > \|\|\ + > A C A + > EOF + rebasing 2:86d01f49c0d9 "B" (B) + rebasing 3:b7df2ca01aa8 "D" (D) + o 4: D + |\ + +---o 3: B + | |/ + | o 2: Z + | | + o | 1: C + / + o 0: A + diff -Nru mercurial-4.0.2~trusty1/tests/test-rebase-collapse.t mercurial-4.1~rc~trusty1/tests/test-rebase-collapse.t --- mercurial-4.0.2~trusty1/tests/test-rebase-collapse.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rebase-collapse.t 2017-01-18 16:54:51.000000000 +0000 @@ -836,7 +836,7 @@ rebasing 2:b8d8db2b242d "a-dev" (tip) saved backup bundle to $TESTTMP/collapse_remember_message/.hg/strip-backup/b8d8db2b242d-f474c19a-backup.hg (glob) $ hg log - changeset: 2:12bb766dceb1 + changeset: 2:45ba1d1a8665 tag: tip user: test date: Thu Jan 01 00:00:00 1970 +0000 diff -Nru mercurial-4.0.2~trusty1/tests/test-rebase-conflicts.t mercurial-4.1~rc~trusty1/tests/test-rebase-conflicts.t --- mercurial-4.0.2~trusty1/tests/test-rebase-conflicts.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rebase-conflicts.t 2017-01-18 16:54:51.000000000 +0000 @@ -73,8 +73,6 @@ Try to continue without solving the conflict: $ hg rebase --continue - already rebased 3:3163e20567cc "L1" as 3e046f2ecedb - rebasing 4:46f0b057b5c0 "L2" abort: unresolved merge conflicts (see 'hg help resolve') [255] @@ -335,7 +333,7 @@ @@ -1,2 +1,6 @@ a b - +<<<<<<< dest: 328e4ab1f7cc ab - test: ab + +<<<<<<< dest: 328e4ab1f7cc ab - test: ab +======= +c +>>>>>>> source: 7bc217434fc1 - test: abc @@ -354,7 +352,7 @@ +++ b/a * (glob) @@ -1,2 +1,8 @@ a - +<<<<<<< dest: 328e4ab1f7cc ab - test: ab + +<<<<<<< dest: 328e4ab1f7cc ab - test: ab b +||||||| base +======= diff -Nru mercurial-4.0.2~trusty1/tests/test-rebase-obsolete.t mercurial-4.1~rc~trusty1/tests/test-rebase-obsolete.t --- mercurial-4.0.2~trusty1/tests/test-rebase-obsolete.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rebase-obsolete.t 2017-01-18 16:54:51.000000000 +0000 @@ -761,7 +761,7 @@ o 0:4a2df7238c3b A $ hg summary - parent: 15:73568ab6879d tip + parent: 15:73568ab6879d tip (unstable) bar foo branch: default commit: (clean) diff -Nru mercurial-4.0.2~trusty1/tests/test-rebase-pull.t mercurial-4.1~rc~trusty1/tests/test-rebase-pull.t --- mercurial-4.0.2~trusty1/tests/test-rebase-pull.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rebase-pull.t 2017-01-18 16:54:51.000000000 +0000 @@ -1,6 +1,7 @@ $ cat >> $HGRCPATH < [extensions] > rebase= + > histedit= > > [alias] > tglog = log -G --template "{rev}: '{desc}' {branches}\n" @@ -72,6 +73,63 @@ searching for changes no changes found +Abort pull early if working dir is not clean: + + $ echo L1-mod > L1 + $ hg pull --rebase + abort: uncommitted changes + (cannot pull with rebase: please commit or shelve your changes first) + [255] + $ hg update --clean --quiet + +Abort pull early if another operation (histedit) is in progress: + + $ hg histedit . -q --commands - << EOF + > edit d80cc2da061e histedit: generate unfinished state + > EOF + Editing (d80cc2da061e), you may commit or record as needed now. + (hg histedit --continue to resume) + [1] + $ hg pull --rebase + abort: histedit in progress + (use 'hg histedit --continue' or 'hg histedit --abort') + [255] + $ hg histedit --abort --quiet + +Abort pull early with pending uncommitted merge: + + $ cd .. + $ hg clone --noupdate c d + $ cd d + $ hg tglog + o 1: 'C2' + | + o 0: 'C1' + + $ hg update --quiet 0 + $ echo M1 > M1 + $ hg commit --quiet -Am M1 + $ hg update --quiet 1 + $ hg merge 2 + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + (branch merge, don't forget to commit) + $ hg pull --rebase + abort: outstanding uncommitted merge + (cannot pull with rebase: please commit or shelve your changes first) + [255] + $ hg update --clean --quiet + +Abort pull early with unclean subrepo: + $ echo s = s > .hgsub + $ hg add .hgsub + $ hg init s + $ hg commit -m "generated a subrepo" + $ echo a > s/a + $ hg -R s add s/a + $ hg pull --rebase + abort: uncommitted changes in subrepository 's' + (cannot pull with rebase: please commit or shelve your changes first) + [255] Invoke pull --rebase and nothing to rebase: diff -Nru mercurial-4.0.2~trusty1/tests/test-rebase-scenario-global.t mercurial-4.1~rc~trusty1/tests/test-rebase-scenario-global.t --- mercurial-4.0.2~trusty1/tests/test-rebase-scenario-global.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rebase-scenario-global.t 2017-01-18 16:54:51.000000000 +0000 @@ -838,7 +838,7 @@ $ cd .. -Make the repo a bit more interresting +Make the repo a bit more interesting $ hg up 1 0 files updated, 0 files merged, 2 files removed, 0 files unresolved diff -Nru mercurial-4.0.2~trusty1/tests/test-rename-dir-merge.t mercurial-4.1~rc~trusty1/tests/test-rename-dir-merge.t --- mercurial-4.0.2~trusty1/tests/test-rename-dir-merge.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-rename-dir-merge.t 2017-01-18 16:54:51.000000000 +0000 @@ -148,7 +148,7 @@ target ======= baz - >>>>>>> merge rev: ce36d17b18fb - test: 2 add a/c + >>>>>>> merge rev: ce36d17b18fb - test: 2 add a/c $ rm b/c.orig Remote directory rename with conflicting file added in remote target directory @@ -177,7 +177,7 @@ ? a/d ? b/c.orig $ cat b/c - <<<<<<< working copy: ce36d17b18fb - test: 2 add a/c + <<<<<<< working copy: ce36d17b18fb - test: 2 add a/c baz ======= target diff -Nru mercurial-4.0.2~trusty1/tests/test-repo-compengines.t mercurial-4.1~rc~trusty1/tests/test-repo-compengines.t --- mercurial-4.0.2~trusty1/tests/test-repo-compengines.t 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-repo-compengines.t 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,78 @@ +A new repository uses zlib storage, which doesn't need a requirement + + $ hg init default + $ cd default + $ cat .hg/requires + dotencode + fncache + generaldelta + revlogv1 + store + + $ touch foo + $ hg -q commit -A -m 'initial commit with a lot of repeated repeated repeated text to trigger compression' + $ hg debugrevlog -c | grep 0x78 + 0x78 (x) : 1 (100.00%) + 0x78 (x) : 110 (100.00%) + + $ cd .. + +Unknown compression engine to format.compression aborts + + $ hg --config experimental.format.compression=unknown init unknown + abort: compression engine unknown defined by experimental.format.compression not available + (run "hg debuginstall" to list available compression engines) + [255] + +A requirement specifying an unknown compression engine results in bail + + $ hg init unknownrequirement + $ cd unknownrequirement + $ echo exp-compression-unknown >> .hg/requires + $ hg log + abort: repository requires features unknown to this Mercurial: exp-compression-unknown! + (see https://mercurial-scm.org/wiki/MissingRequirement for more information) + [255] + + $ cd .. + +#if zstd + + $ hg --config experimental.format.compression=zstd init zstd + $ cd zstd + $ cat .hg/requires + dotencode + exp-compression-zstd + fncache + generaldelta + revlogv1 + store + + $ touch foo + $ hg -q commit -A -m 'initial commit with a lot of repeated repeated repeated text' + + $ hg debugrevlog -c | grep 0x28 + 0x28 : 1 (100.00%) + 0x28 : 98 (100.00%) + + $ cd .. + +Specifying a new format.compression on an existing repo won't introduce data +with that engine or a requirement + + $ cd default + $ touch bar + $ hg --config experimental.format.compression=zstd -q commit -A -m 'add bar with a lot of repeated repeated repeated text' + + $ cat .hg/requires + dotencode + fncache + generaldelta + revlogv1 + store + + $ hg debugrevlog -c | grep 0x78 + 0x78 (x) : 2 (100.00%) + 0x78 (x) : 199 (100.00%) + +#endif diff -Nru mercurial-4.0.2~trusty1/tests/test-revert-interactive.t mercurial-4.1~rc~trusty1/tests/test-revert-interactive.t --- mercurial-4.0.2~trusty1/tests/test-revert-interactive.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-revert-interactive.t 2017-01-18 16:54:51.000000000 +0000 @@ -46,6 +46,7 @@ > y > y > y + > y > n > n > EOF @@ -53,6 +54,7 @@ reverting folder1/g (glob) removing folder1/i (glob) reverting folder2/h (glob) + remove added file folder1/i (Yn)? y diff --git a/f b/f 2 hunks, 2 lines changed examine changes to 'f'? [Ynesfdaq?] y @@ -137,7 +139,7 @@ $ ls folder1/ g -Test that a noop revert doesn't do an unecessary backup +Test that a noop revert doesn't do an unnecessary backup $ (echo y; echo n) | hg revert -i -r 2 folder1/g diff --git a/folder1/g b/folder1/g 1 hunks, 1 lines changed @@ -174,6 +176,7 @@ $ hg update -C 6 3 files updated, 0 files merged, 0 files removed, 0 files unresolved $ hg revert -i -r 2 --all -- << EOF + > n > y > y > y @@ -186,6 +189,7 @@ reverting folder1/g (glob) removing folder1/i (glob) reverting folder2/h (glob) + remove added file folder1/i (Yn)? n diff --git a/f b/f 2 hunks, 2 lines changed examine changes to 'f'? [Ynesfdaq?] y @@ -258,7 +262,6 @@ $ hg st M f M folder1/g - R folder1/i $ hg revert --interactive f << EOF > y > y @@ -290,7 +293,6 @@ $ hg st M f M folder1/g - R folder1/i ? f.orig $ cat f a @@ -307,7 +309,7 @@ 5 $ rm f.orig $ hg update -C . - 3 files updated, 0 files merged, 0 files removed, 0 files unresolved + 2 files updated, 0 files merged, 0 files removed, 0 files unresolved Check editing files newly added by a revert @@ -424,14 +426,14 @@ > n > EOF forgetting newfile - forget added file newfile (yn)? n + forget added file newfile (Yn)? n $ hg status A newfile $ hg revert -i < y > EOF forgetting newfile - forget added file newfile (yn)? y + forget added file newfile (Yn)? y $ hg status ? newfile diff -Nru mercurial-4.0.2~trusty1/tests/test-revert.t mercurial-4.1~rc~trusty1/tests/test-revert.t --- mercurial-4.0.2~trusty1/tests/test-revert.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-revert.t 2017-01-18 16:54:51.000000000 +0000 @@ -784,7 +784,7 @@ (setup from reference repo) - $ cp -r revert-ref revert-parent-all + $ cp -R revert-ref revert-parent-all $ cd revert-parent-all check revert output @@ -841,7 +841,7 @@ (setup from reference repo) - $ cp -r revert-ref revert-base-all + $ cp -R revert-ref revert-base-all $ cd revert-base-all check revert output @@ -896,7 +896,7 @@ (setup from reference repo) - $ cp -r revert-ref revert-parent-explicit + $ cp -R revert-ref revert-parent-explicit $ cd revert-parent-explicit revert all files individually and check the output @@ -989,7 +989,7 @@ (setup from reference repo) - $ cp -r revert-ref revert-base-explicit + $ cp -R revert-ref revert-base-explicit $ cd revert-base-explicit revert all files individually and check the output diff -Nru mercurial-4.0.2~trusty1/tests/test-revlog-ancestry.py mercurial-4.1~rc~trusty1/tests/test-revlog-ancestry.py --- mercurial-4.0.2~trusty1/tests/test-revlog-ancestry.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-revlog-ancestry.py 2017-01-18 16:54:51.000000000 +0000 @@ -6,7 +6,7 @@ ui as uimod, ) -u = uimod.ui() +u = uimod.ui.load() repo = hg.repository(u, 'test1', create=1) os.chdir('test1') diff -Nru mercurial-4.0.2~trusty1/tests/test-revset.t mercurial-4.1~rc~trusty1/tests/test-revset.t --- mercurial-4.0.2~trusty1/tests/test-revset.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-revset.t 2017-01-18 16:54:51.000000000 +0000 @@ -161,8 +161,8 @@ (rangeall None) * optimized: - (rangepre - ('string', 'tip') + (rangeall + None define) * set: @@ -619,8 +619,8 @@ (rangeall None) * analyzed: - (rangepre - ('string', 'tip') + (rangeall + None define) * set: @@ -865,6 +865,17 @@ 7 8 9 + $ log 'author(r"re:\S")' + 0 + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 $ log 'branch(é)' 8 9 @@ -881,6 +892,13 @@ $ log 'children(ancestor(4,5))' 2 3 + + $ log 'children(4)' + 6 + 8 + $ log 'children(null)' + 0 + $ log 'closed()' $ log 'contains(a)' 0 @@ -894,6 +912,9 @@ 5 $ log 'desc(B)' 5 + $ hg log -r 'desc(r"re:S?u")' --template "{rev} {desc|firstline}\n" + 5 5 bug + 6 6 issue619 $ log 'descendants(2 or 3)' 2 3 @@ -1079,7 +1100,7 @@ 8 9 -Test opreand of '%' is optimized recursively (issue4670) +Test operand of '%' is optimized recursively (issue4670) $ try --optimize '8:9-8%' (onlypost @@ -2802,8 +2823,8 @@ [255] Undocumented functions aren't suggested as similar either - $ log 'wdir2()' - hg: parse error: unknown identifier: wdir2 + $ log 'tagged2()' + hg: parse error: unknown identifier: tagged2 [255] multiple revspecs @@ -3085,7 +3106,7 @@ [255] test scope of alias expansion: 'universe' is expanded prior to 'shadowall(0)', -but 'all()' should never be substituded to '0()'. +but 'all()' should never be substituted to '0()'. $ echo 'universe = all()' >> .hg/hgrc $ echo 'shadowall(all) = all and universe' >> .hg/hgrc diff -Nru mercurial-4.0.2~trusty1/tests/test-setdiscovery.t mercurial-4.1~rc~trusty1/tests/test-setdiscovery.t --- mercurial-4.0.2~trusty1/tests/test-setdiscovery.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-setdiscovery.t 2017-01-18 16:54:51.000000000 +0000 @@ -349,9 +349,9 @@ $ killdaemons.py $ cut -d' ' -f6- access.log | grep -v cmd=known # cmd=known uses random sampling "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D513314ca8b3ae4dac8eec56966265b00fcf866db - "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=513314ca8b3ae4dac8eec56966265b00fcf866db&heads=e64a39e7da8b0d54bc63e81169aff001c13b3477 - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases + "GET /?cmd=batch HTTP/1.1" 200 - x-hgarg-1:cmds=heads+%3Bknown+nodes%3D513314ca8b3ae4dac8eec56966265b00fcf866db x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=getbundle HTTP/1.1" 200 - x-hgarg-1:bundlecaps=HG20%2Cbundle2%3DHG20%250Achangegroup%253D01%252C02%250Adigests%253Dmd5%252Csha1%252Csha512%250Aerror%253Dabort%252Cunsupportedcontent%252Cpushraced%252Cpushkey%250Ahgtagsfnodes%250Alistkeys%250Apushkey%250Aremote-changegroup%253Dhttp%252Chttps&cg=1&common=513314ca8b3ae4dac8eec56966265b00fcf866db&heads=e64a39e7da8b0d54bc63e81169aff001c13b3477 x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) $ cat errors.log $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-shelve.t mercurial-4.1~rc~trusty1/tests/test-shelve.t --- mercurial-4.0.2~trusty1/tests/test-shelve.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-shelve.t 2017-01-18 16:54:51.000000000 +0000 @@ -100,6 +100,46 @@ default.hg default.patch +checks to make sure we dont create a directory or +hidden file while choosing a new shelve name + +when we are given a name + + $ hg shelve -n foo/bar + abort: shelved change names can not contain slashes + [255] + $ hg shelve -n .baz + abort: shelved change names can not start with '.' + [255] + $ hg shelve -n foo\\bar + abort: shelved change names can not contain slashes + [255] + +when shelve has to choose itself + + $ hg branch x/y -q + $ hg commit -q -m "Branch commit 0" + $ hg shelve + nothing changed + [1] + $ hg branch .x -q + $ hg commit -q -m "Branch commit 1" + $ hg shelve + nothing changed + [1] + $ hg branch x\\y -q + $ hg commit -q -m "Branch commit 2" + $ hg shelve + nothing changed + [1] + +cleaning the branches made for name checking tests + + $ hg up default -q + $ hg strip 3 -q + $ hg strip 2 -q + $ hg strip 1 -q + create an mq patch - shelving should work fine with a patch applied $ echo n > n @@ -128,15 +168,6 @@ c R b/b -prevent some foot-shooting - - $ hg shelve -n foo/bar - abort: shelved change names may not contain slashes - [255] - $ hg shelve -n .baz - abort: shelved change names may not start with '.' - [255] - the common case - no options or filenames $ hg shelve @@ -195,12 +226,12 @@ (this also tests that same timestamp prevents backups from being removed, even though there are more than 'maxbackups' backups) - $ f -t .hg/shelve-backup/default.hg - .hg/shelve-backup/default.hg: file - $ touch -t 200001010000 .hg/shelve-backup/default.hg - $ f -t .hg/shelve-backup/default-1.hg - .hg/shelve-backup/default-1.hg: file - $ touch -t 200001010000 .hg/shelve-backup/default-1.hg + $ f -t .hg/shelve-backup/default.patch + .hg/shelve-backup/default.patch: file + $ touch -t 200001010000 .hg/shelve-backup/default.patch + $ f -t .hg/shelve-backup/default-1.patch + .hg/shelve-backup/default-1.patch: file + $ touch -t 200001010000 .hg/shelve-backup/default-1.patch $ hg unshelve unshelving change 'default-01' @@ -332,7 +363,7 @@ +++ b/a/a @@ -1,2 +1,6 @@ a - +<<<<<<< dest: * - shelve: pending changes temporary commit (glob) + +<<<<<<< dest: * - shelve: pending changes temporary commit (glob) c +======= +a @@ -759,7 +790,7 @@ M f ? f.orig $ cat f - <<<<<<< dest: 5f6b880e719b - shelve: pending changes temporary commit + <<<<<<< dest: 5f6b880e719b - shelve: pending changes temporary commit g ======= f @@ -804,7 +835,7 @@ M f ? f.orig $ cat f - <<<<<<< dest: * - test: intermediate other change (glob) + <<<<<<< dest: * - test: intermediate other change (glob) g ======= f @@ -960,7 +991,7 @@ x x -shelve --patch and shelve --stat should work with a single valid shelfname +shelve --patch and shelve --stat should work with valid shelfnames $ hg up --clean . 1 files updated, 0 files merged, 0 files removed, 0 files unresolved @@ -977,11 +1008,29 @@ shelved as default-01 0 files updated, 0 files merged, 1 files removed, 0 files unresolved $ hg shelve --patch default default-01 - abort: --patch expects a single shelf - [255] + default-01 (*)* changes to: create conflict (glob) + + diff --git a/shelf-patch-b b/shelf-patch-b + new file mode 100644 + --- /dev/null + +++ b/shelf-patch-b + @@ -0,0 +1,1 @@ + +patch b + default (*)* changes to: create conflict (glob) + + diff --git a/shelf-patch-a b/shelf-patch-a + new file mode 100644 + --- /dev/null + +++ b/shelf-patch-a + @@ -0,0 +1,1 @@ + +patch a $ hg shelve --stat default default-01 - abort: --stat expects a single shelf - [255] + default-01 (*)* changes to: create conflict (glob) + shelf-patch-b | 1 + + 1 files changed, 1 insertions(+), 0 deletions(-) + default (*)* changes to: create conflict (glob) + shelf-patch-a | 1 + + 1 files changed, 1 insertions(+), 0 deletions(-) $ hg shelve --patch default default (*)* changes to: create conflict (glob) @@ -1001,6 +1050,12 @@ $ hg shelve --stat nonexistentshelf abort: cannot find shelf nonexistentshelf [255] + $ hg shelve --patch default nonexistentshelf + abort: cannot find shelf nonexistentshelf + [255] + $ hg shelve --patch + abort: --patch expects at least one shelf + [255] $ cd .. @@ -1383,6 +1438,7 @@ 0 files updated, 0 files merged, 1 files removed, 0 files unresolved $ hg branch default + $ cd .. When i shelve commit on newly created branch i expect that after unshelve newly created branch will be preserved. @@ -1416,6 +1472,7 @@ ? b $ hg branch test + $ cd .. When i shelve commit on newly created branch, make some changes, unshelve it and running into merge @@ -1489,11 +1546,12 @@ A b $ hg branch default + $ cd .. When i unshelve resulting in merge conflicts and makes saved file shelvedstate looks like in previous versions in mercurial(without restore branch information in 7th line) i -expect that after resolving conflicts and succesfully +expect that after resolving conflicts and successfully running 'shelve --continue' the branch information won't be restored and branch will be unchanged. @@ -1551,6 +1609,7 @@ M a $ hg branch default + $ cd .. On non bare shelve the branch information shouldn't be restored @@ -1587,7 +1646,7 @@ default $ cd .. -Prepare unshleve with a corrupted shelvedstate +Prepare unshelve with a corrupted shelvedstate $ hg init r1 && cd r1 $ echo text1 > file && hg add file $ hg shelve @@ -1622,3 +1681,32 @@ abort: no unshelve in progress [255] $ cd .. + +Unshelve respects --keep even if user intervention is needed + $ hg init unshelvekeep && cd unshelvekeep + $ echo 1 > file && hg ci -Am 1 + adding file + $ echo 2 >> file + $ hg shelve + shelved as default + 1 files updated, 0 files merged, 0 files removed, 0 files unresolved + $ echo 3 >> file && hg ci -Am 13 + $ hg shelve --list + default (1s ago) changes to: 1 + $ hg unshelve --keep + unshelving change 'default' + rebasing shelved changes + rebasing 2:3fbe6fbb0bef "changes to: 1" (tip) + merging file + warning: conflicts while merging file! (edit, then use 'hg resolve --mark') + unresolved conflicts (see 'hg resolve', then 'hg unshelve --continue') + [1] + $ hg resolve --mark file + (no more unresolved files) + continue: hg unshelve --continue + $ hg unshelve --continue + rebasing 2:3fbe6fbb0bef "changes to: 1" (tip) + unshelve of 'default' complete + $ hg shelve --list + default (*s ago) changes to: 1 (glob) + $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-ssh-bundle1.t mercurial-4.1~rc~trusty1/tests/test-ssh-bundle1.t --- mercurial-4.0.2~trusty1/tests/test-ssh-bundle1.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-ssh-bundle1.t 2017-01-18 16:54:51.000000000 +0000 @@ -464,8 +464,8 @@ running python ".*/dummyssh" user@dummy ('|")hg -R remote serve --stdio('|") (re) sending hello command sending between command - remote: 371 - remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024 + remote: 355 + remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN remote: 1 preparing listkeys for "bookmarks" sending listkeys command diff -Nru mercurial-4.0.2~trusty1/tests/test-ssh.t mercurial-4.1~rc~trusty1/tests/test-ssh.t --- mercurial-4.0.2~trusty1/tests/test-ssh.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-ssh.t 2017-01-18 16:54:51.000000000 +0000 @@ -265,8 +265,17 @@ > sys.stdout.write("KABOOM\n") > EOF - $ echo '[hooks]' >> ../remote/.hg/hgrc - $ echo "changegroup.stdout = python $TESTTMP/badhook" >> ../remote/.hg/hgrc + $ cat < $TESTTMP/badpyhook.py + > import sys + > def hook(ui, repo, hooktype, **kwargs): + > sys.stdout.write("KABOOM IN PROCESS\n") + > EOF + + $ cat <> ../remote/.hg/hgrc + > [hooks] + > changegroup.stdout = python $TESTTMP/badhook + > changegroup.pystdout = python:$TESTTMP/badpyhook.py:hook + > EOF $ echo r > r $ hg ci -A -m z r @@ -281,6 +290,7 @@ remote: adding file changes remote: added 1 changesets with 1 changes to 1 files remote: KABOOM + remote: KABOOM IN PROCESS $ hg -R ../remote heads changeset: 5:1383141674ec tag: tip @@ -447,6 +457,7 @@ remote: adding file changes remote: added 1 changesets with 1 changes to 1 files remote: KABOOM + remote: KABOOM IN PROCESS local stdout debug output @@ -456,8 +467,8 @@ running python ".*/dummyssh" user@dummy ('|")hg -R remote serve --stdio('|") (re) sending hello command sending between command - remote: 371 - remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN httpheader=1024 + remote: 355 + remote: capabilities: lookup changegroupsubset branchmap pushkey known getbundle unbundlehash batch streamreqs=generaldelta,revlogv1 bundle2=HG20%0Achangegroup%3D01%2C02%0Adigests%3Dmd5%2Csha1%2Csha512%0Aerror%3Dabort%2Cunsupportedcontent%2Cpushraced%2Cpushkey%0Ahgtagsfnodes%0Alistkeys%0Apushkey%0Aremote-changegroup%3Dhttp%2Chttps unbundle=HG10GZ,HG10BZ,HG10UN remote: 1 query 1; heads sending batch command diff -Nru mercurial-4.0.2~trusty1/tests/test-status-inprocess.py mercurial-4.1~rc~trusty1/tests/test-status-inprocess.py --- mercurial-4.0.2~trusty1/tests/test-status-inprocess.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-status-inprocess.py 2017-01-18 16:54:51.000000000 +0000 @@ -7,7 +7,7 @@ ui as uimod, ) -u = uimod.ui() +u = uimod.ui.load() print('% creating repo') repo = localrepo.localrepository(u, '.', create=True) diff -Nru mercurial-4.0.2~trusty1/tests/test-subrepo.t mercurial-4.1~rc~trusty1/tests/test-subrepo.t --- mercurial-4.0.2~trusty1/tests/test-subrepo.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-subrepo.t 2017-01-18 16:54:51.000000000 +0000 @@ -58,10 +58,10 @@ $ mkdir snot $ touch snot/file $ hg remove -S snot/file - not removing snot/file: file is untracked + not removing snot/file: file is untracked (glob) [1] $ hg cat snot/filenot - snot/filenot: no such file in rev 7cf8cfea66e4 + snot/filenot: no such file in rev 7cf8cfea66e4 (glob) [1] $ rm -r snot @@ -332,7 +332,7 @@ conflict ======= t3 - >>>>>>> other: 7af322bc1198 - test: 7 + >>>>>>> other: 7af322bc1198 - test: 7 11: remove subrepo t @@ -1518,8 +1518,8 @@ Courtesy phases synchronisation to publishing server does not block the push (issue3781) - $ cp -r main issue3781 - $ cp -r main issue3781-dest + $ cp -R main issue3781 + $ cp -R main issue3781-dest $ cd issue3781-dest/s $ hg phase tip # show we have draft changeset 5: draft diff -Nru mercurial-4.0.2~trusty1/tests/test-symlink-os-yes-fs-no.py mercurial-4.1~rc~trusty1/tests/test-symlink-os-yes-fs-no.py --- mercurial-4.0.2~trusty1/tests/test-symlink-os-yes-fs-no.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-symlink-os-yes-fs-no.py 2017-01-18 16:54:51.000000000 +0000 @@ -17,7 +17,7 @@ if not getattr(os, "symlink", False): sys.exit(80) # SKIPPED_STATUS defined in run-tests.py -u = uimod.ui() +u = uimod.ui.load() # hide outer repo hg.peer(u, {}, '.', create=True) @@ -35,6 +35,9 @@ def symlink_failure(src, dst): raise OSError(1, "Operation not permitted") os.symlink = symlink_failure +def islink_failure(path): + return False +os.path.islink = islink_failure # dereference links as if a Samba server has exported this to a # Windows client @@ -45,10 +48,10 @@ fp.close() # reload repository -u = uimod.ui() +u = uimod.ui.load() repo = hg.repository(u, 'test0') commands.status(u, repo) # try cloning a repo which contains symlinks -u = uimod.ui() +u = uimod.ui.load() hg.clone(u, {}, BUNDLEPATH, 'test1') diff -Nru mercurial-4.0.2~trusty1/tests/test-tags.t mercurial-4.1~rc~trusty1/tests/test-tags.t --- mercurial-4.0.2~trusty1/tests/test-tags.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-tags.t 2017-01-18 16:54:51.000000000 +0000 @@ -672,6 +672,9 @@ $ ls tagsclient/.hg/cache branch2-served + checkisexec + checklink + checklink-target hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 @@ -696,6 +699,9 @@ $ ls tagsclient/.hg/cache branch2-served + checkisexec + checklink + checklink-target hgtagsfnodes1 rbc-names-v1 rbc-revs-v1 diff -Nru mercurial-4.0.2~trusty1/tests/test-transplant.t mercurial-4.1~rc~trusty1/tests/test-transplant.t --- mercurial-4.0.2~trusty1/tests/test-transplant.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-transplant.t 2017-01-18 16:54:51.000000000 +0000 @@ -102,9 +102,10 @@ 7 b3 $ hg log -r 'transplanted(head())' --template '{rev} {parents} {desc}\n' 7 b3 - $ hg help revsets | grep transplanted + $ hg help revisions.transplanted "transplanted([set])" Transplanted changesets in set, or all transplanted changesets. + test transplanted keyword diff -Nru mercurial-4.0.2~trusty1/tests/test-treediscovery.t mercurial-4.1~rc~trusty1/tests/test-treediscovery.t --- mercurial-4.0.2~trusty1/tests/test-treediscovery.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-treediscovery.t 2017-01-18 16:54:51.000000000 +0000 @@ -503,34 +503,68 @@ no changes found [1] $ cd .. +#if zstd $ tstop show "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=heads HTTP/1.1" 200 - - "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 - "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 - "GET /?cmd=changegroupsubset HTTP/1.1" 200 - x-hgarg-1:bases=d8f638ac69e9ae8dea4f09f11d696546a912d961&heads=d8f638ac69e9ae8dea4f09f11d696546a912d961 - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=heads HTTP/1.1" 200 - - "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 - "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=heads HTTP/1.1" 200 - - "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 - "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 - "GET /?cmd=changegroupsubset HTTP/1.1" 200 - x-hgarg-1:bases=d8f638ac69e9ae8dea4f09f11d696546a912d961&heads=d8f638ac69e9ae8dea4f09f11d696546a912d961 - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=heads HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=branchmap HTTP/1.1" 200 - - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=changegroupsubset HTTP/1.1" 200 - x-hgarg-1:bases=d8f638ac69e9ae8dea4f09f11d696546a912d961&heads=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=changegroupsubset HTTP/1.1" 200 - x-hgarg-1:bases=d8f638ac69e9ae8dea4f09f11d696546a912d961&heads=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 "POST /?cmd=unbundle HTTP/1.1" 200 - x-hgarg-1:heads=686173686564+1827a5bb63e602382eb89dd58f2ac9f3b007ad91* (glob) - "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=heads HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 "GET /?cmd=capabilities HTTP/1.1" 200 - - "GET /?cmd=heads HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zstd,zlib,none,bzip2 +#else + $ tstop show + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=changegroupsubset HTTP/1.1" 200 - x-hgarg-1:bases=d8f638ac69e9ae8dea4f09f11d696546a912d961&heads=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=branches HTTP/1.1" 200 - x-hgarg-1:nodes=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=between HTTP/1.1" 200 - x-hgarg-1:pairs=d8f638ac69e9ae8dea4f09f11d696546a912d961-d57206cc072a18317c1e381fb60aa31bd3401785 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=changegroupsubset HTTP/1.1" 200 - x-hgarg-1:bases=d8f638ac69e9ae8dea4f09f11d696546a912d961&heads=d8f638ac69e9ae8dea4f09f11d696546a912d961 x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=branchmap HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=bookmarks x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "POST /?cmd=unbundle HTTP/1.1" 200 - x-hgarg-1:heads=686173686564+1827a5bb63e602382eb89dd58f2ac9f3b007ad91* (glob) + "GET /?cmd=listkeys HTTP/1.1" 200 - x-hgarg-1:namespace=phases x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 + "GET /?cmd=capabilities HTTP/1.1" 200 - + "GET /?cmd=heads HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=zlib,none,bzip2 +#endif diff -Nru mercurial-4.0.2~trusty1/tests/test-treemanifest.t mercurial-4.1~rc~trusty1/tests/test-treemanifest.t --- mercurial-4.0.2~trusty1/tests/test-treemanifest.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-treemanifest.t 2017-01-18 16:54:51.000000000 +0000 @@ -458,7 +458,7 @@ b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) - $ cp -r .hg/store .hg/store-copy + $ cp -R .hg/store .hg/store-copy Test files for a subdirectory. @@ -468,7 +468,7 @@ b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) b/foo/apple/bees/flower.py (glob) - $ cp -r .hg/store-copy/. .hg/store + $ cp -R .hg/store-copy/. .hg/store Test files with just includes and excludes. @@ -477,7 +477,7 @@ $ rm -r .hg/store/meta/b/foo/apple/bees $ hg files -r . -I path:b/bar -X path:b/bar/orange/fly -I path:b/foo -X path:b/foo/apple/bees b/bar/fruits.txt (glob) - $ cp -r .hg/store-copy/. .hg/store + $ cp -R .hg/store-copy/. .hg/store Test files for a subdirectory, excluding a directory within it. @@ -487,7 +487,7 @@ b/bar/fruits.txt (glob) b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) - $ cp -r .hg/store-copy/. .hg/store + $ cp -R .hg/store-copy/. .hg/store Test files for a sub directory, including only a directory within it, and including an unrelated directory. @@ -497,7 +497,7 @@ $ hg files -r . -I path:b/bar/orange -I path:a b b/bar/orange/fly/gnat.py (glob) b/bar/orange/fly/housefly.txt (glob) - $ cp -r .hg/store-copy/. .hg/store + $ cp -R .hg/store-copy/. .hg/store Test files for a pattern, including a directory, and excluding a directory within that. @@ -507,7 +507,7 @@ $ rm -r .hg/store/meta/b/bar/orange $ hg files -r . glob:**.txt -I path:b/bar -X path:b/bar/orange b/bar/fruits.txt (glob) - $ cp -r .hg/store-copy/. .hg/store + $ cp -R .hg/store-copy/. .hg/store Add some more changes to the deep repo $ echo narf >> b/bar/fruits.txt @@ -553,7 +553,7 @@ $ killdaemons.py Back up the recently added revlogs - $ cp -r .hg/store .hg/store-newcopy + $ cp -R .hg/store .hg/store-newcopy Verify reports missing dirlog $ rm .hg/store/meta/b/00manifest.* @@ -582,7 +582,7 @@ 8 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-newcopy/. .hg/store + $ cp -R .hg/store-newcopy/. .hg/store Verify reports missing dirlog entry $ mv -f .hg/store-copy/meta/b/00manifest.* .hg/store/meta/b/ @@ -607,7 +607,7 @@ 8 integrity errors encountered! (first damaged changeset appears to be 1) [1] - $ cp -r .hg/store-newcopy/. .hg/store + $ cp -R .hg/store-newcopy/. .hg/store Test cloning a treemanifest repo over http. $ hg serve -p $HGPORT -d --pid-file=hg.pid --errorlog=errors.log diff -Nru mercurial-4.0.2~trusty1/tests/test-trusted.py mercurial-4.1~rc~trusty1/tests/test-trusted.py --- mercurial-4.0.2~trusty1/tests/test-trusted.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-trusted.py 2017-01-18 16:54:51.000000000 +0000 @@ -66,7 +66,7 @@ print('# %s user, %s group%s' % (kind[user == cuser], kind[group == cgroup], trusted)) - u = uimod.ui() + u = uimod.ui.load() u.setconfig('ui', 'debug', str(bool(debug))) u.setconfig('ui', 'report_untrusted', str(bool(report))) u.readconfig('.hg/hgrc') @@ -156,7 +156,7 @@ print() print("# read trusted, untrusted, new ui, trusted") -u = uimod.ui() +u = uimod.ui.load() u.setconfig('ui', 'debug', 'on') u.readconfig(filename) u2 = u.copy() diff -Nru mercurial-4.0.2~trusty1/tests/test-ui-color.py mercurial-4.1~rc~trusty1/tests/test-ui-color.py --- mercurial-4.0.2~trusty1/tests/test-ui-color.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-ui-color.py 2017-01-18 16:54:51.000000000 +0000 @@ -23,7 +23,7 @@ hgrc.write('color=\n') hgrc.close() -ui_ = uimod.ui() +ui_ = uimod.ui.load() ui_.setconfig('ui', 'formatted', 'True') # we're not interested in the output, so write that to devnull diff -Nru mercurial-4.0.2~trusty1/tests/test-ui-config.py mercurial-4.1~rc~trusty1/tests/test-ui-config.py --- mercurial-4.0.2~trusty1/tests/test-ui-config.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-ui-config.py 2017-01-18 16:54:51.000000000 +0000 @@ -5,7 +5,7 @@ ui as uimod, ) -testui = uimod.ui() +testui = uimod.ui.load() parsed = dispatch._parseconfig(testui, [ 'values.string=string value', 'values.bool1=true', diff -Nru mercurial-4.0.2~trusty1/tests/test-ui-verbosity.py mercurial-4.1~rc~trusty1/tests/test-ui-verbosity.py --- mercurial-4.0.2~trusty1/tests/test-ui-verbosity.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-ui-verbosity.py 2017-01-18 16:54:51.000000000 +0000 @@ -32,7 +32,7 @@ f.write('debug = True\n') f.close() - u = uimod.ui() + u = uimod.ui.load() if cmd_quiet or cmd_debug or cmd_verbose: u.setconfig('ui', 'quiet', str(bool(cmd_quiet))) u.setconfig('ui', 'verbose', str(bool(cmd_verbose))) diff -Nru mercurial-4.0.2~trusty1/tests/test-upgrade-repo.t mercurial-4.1~rc~trusty1/tests/test-upgrade-repo.t --- mercurial-4.0.2~trusty1/tests/test-upgrade-repo.t 1970-01-01 00:00:00.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-upgrade-repo.t 2017-01-18 16:54:51.000000000 +0000 @@ -0,0 +1,312 @@ + $ cat >> $HGRCPATH << EOF + > [extensions] + > share = + > EOF + +store and revlogv1 are required in source + + $ hg --config format.usestore=false init no-store + $ hg -R no-store debugupgraderepo + abort: cannot upgrade repository; requirement missing: store + [255] + + $ hg init no-revlogv1 + $ cat > no-revlogv1/.hg/requires << EOF + > dotencode + > fncache + > generaldelta + > store + > EOF + + $ hg -R no-revlogv1 debugupgraderepo + abort: cannot upgrade repository; requirement missing: revlogv1 + [255] + +Cannot upgrade shared repositories + + $ hg init share-parent + $ hg -q share share-parent share-child + + $ hg -R share-child debugupgraderepo + abort: cannot upgrade repository; unsupported source requirement: shared + [255] + +Do not yet support upgrading manifestv2 and treemanifest repos + + $ hg --config experimental.manifestv2=true init manifestv2 + $ hg -R manifestv2 debugupgraderepo + abort: cannot upgrade repository; unsupported source requirement: manifestv2 + [255] + + $ hg --config experimental.treemanifest=true init treemanifest + $ hg -R treemanifest debugupgraderepo + abort: cannot upgrade repository; unsupported source requirement: treemanifest + [255] + +Cannot add manifestv2 or treemanifest requirement during upgrade + + $ hg init disallowaddedreq + $ hg -R disallowaddedreq --config experimental.manifestv2=true --config experimental.treemanifest=true debugupgraderepo + abort: cannot upgrade repository; do not support adding requirement: manifestv2, treemanifest + [255] + +An upgrade of a repository created with recommended settings only suggests optimizations + + $ hg init empty + $ cd empty + $ hg debugupgraderepo + (no feature deficiencies found in existing repository) + performing an upgrade with "--run" will make the following changes: + + requirements + preserved: dotencode, fncache, generaldelta, revlogv1, store + + additional optimizations are available by specifying "--optimize ": + + redeltaparent + deltas within internal storage will be recalculated to choose an optimal base revision where this was not already done; the size of the repository may shrink and various operations may become faster; the first time this optimization is performed could slow down upgrade execution considerably; subsequent invocations should not run noticeably slower + + redeltamultibase + deltas within internal storage will be recalculated against multiple base revision and the smallest difference will be used; the size of the repository may shrink significantly when there are many merges; this optimization will slow down execution in proportion to the number of merges in the repository and the amount of files in the repository; this slow down should not be significant unless there are tens of thousands of files and thousands of merges + + redeltaall + deltas within internal storage will always be recalculated without reusing prior deltas; this will likely make execution run several times slower; this optimization is typically not needed + + +--optimize can be used to add optimizations + + $ hg debugupgrade --optimize redeltaparent + (no feature deficiencies found in existing repository) + performing an upgrade with "--run" will make the following changes: + + requirements + preserved: dotencode, fncache, generaldelta, revlogv1, store + + redeltaparent + deltas within internal storage will choose a new base revision if needed + + additional optimizations are available by specifying "--optimize ": + + redeltamultibase + deltas within internal storage will be recalculated against multiple base revision and the smallest difference will be used; the size of the repository may shrink significantly when there are many merges; this optimization will slow down execution in proportion to the number of merges in the repository and the amount of files in the repository; this slow down should not be significant unless there are tens of thousands of files and thousands of merges + + redeltaall + deltas within internal storage will always be recalculated without reusing prior deltas; this will likely make execution run several times slower; this optimization is typically not needed + + +Various sub-optimal detections work + + $ cat > .hg/requires << EOF + > revlogv1 + > store + > EOF + + $ hg debugupgraderepo + repository lacks features recommended by current config options: + + fncache + long and reserved filenames may not work correctly; repository performance is sub-optimal + + dotencode + storage of filenames beginning with a period or space may not work correctly + + generaldelta + deltas within internal storage are unable to choose optimal revisions; repository is larger and slower than it could be; interaction with other repositories may require extra network and CPU resources, making "hg push" and "hg pull" slower + + + performing an upgrade with "--run" will make the following changes: + + requirements + preserved: revlogv1, store + added: dotencode, fncache, generaldelta + + fncache + repository will be more resilient to storing certain paths and performance of certain operations should be improved + + dotencode + repository will be better able to store files beginning with a space or period + + generaldelta + repository storage will be able to create optimal deltas; new repository data will be smaller and read times should decrease; interacting with other repositories using this storage model should require less network and CPU resources, making "hg push" and "hg pull" faster + + additional optimizations are available by specifying "--optimize ": + + redeltaparent + deltas within internal storage will be recalculated to choose an optimal base revision where this was not already done; the size of the repository may shrink and various operations may become faster; the first time this optimization is performed could slow down upgrade execution considerably; subsequent invocations should not run noticeably slower + + redeltamultibase + deltas within internal storage will be recalculated against multiple base revision and the smallest difference will be used; the size of the repository may shrink significantly when there are many merges; this optimization will slow down execution in proportion to the number of merges in the repository and the amount of files in the repository; this slow down should not be significant unless there are tens of thousands of files and thousands of merges + + redeltaall + deltas within internal storage will always be recalculated without reusing prior deltas; this will likely make execution run several times slower; this optimization is typically not needed + + + $ hg --config format.dotencode=false debugupgraderepo + repository lacks features recommended by current config options: + + fncache + long and reserved filenames may not work correctly; repository performance is sub-optimal + + generaldelta + deltas within internal storage are unable to choose optimal revisions; repository is larger and slower than it could be; interaction with other repositories may require extra network and CPU resources, making "hg push" and "hg pull" slower + + repository lacks features used by the default config options: + + dotencode + storage of filenames beginning with a period or space may not work correctly + + + performing an upgrade with "--run" will make the following changes: + + requirements + preserved: revlogv1, store + added: fncache, generaldelta + + fncache + repository will be more resilient to storing certain paths and performance of certain operations should be improved + + generaldelta + repository storage will be able to create optimal deltas; new repository data will be smaller and read times should decrease; interacting with other repositories using this storage model should require less network and CPU resources, making "hg push" and "hg pull" faster + + additional optimizations are available by specifying "--optimize ": + + redeltaparent + deltas within internal storage will be recalculated to choose an optimal base revision where this was not already done; the size of the repository may shrink and various operations may become faster; the first time this optimization is performed could slow down upgrade execution considerably; subsequent invocations should not run noticeably slower + + redeltamultibase + deltas within internal storage will be recalculated against multiple base revision and the smallest difference will be used; the size of the repository may shrink significantly when there are many merges; this optimization will slow down execution in proportion to the number of merges in the repository and the amount of files in the repository; this slow down should not be significant unless there are tens of thousands of files and thousands of merges + + redeltaall + deltas within internal storage will always be recalculated without reusing prior deltas; this will likely make execution run several times slower; this optimization is typically not needed + + + $ cd .. + +Upgrading a repository that is already modern essentially no-ops + + $ hg init modern + $ hg -R modern debugupgraderepo --run + upgrade will perform the following actions: + + requirements + preserved: dotencode, fncache, generaldelta, revlogv1, store + + beginning upgrade... + repository locked and read-only + creating temporary repository to stage migrated data: $TESTTMP/modern/.hg/upgrade.* (glob) + (it is safe to interrupt this process any time before data migration completes) + data fully migrated to temporary repository + marking source repository as being upgraded; clients will be unable to read from repository + starting in-place swap of repository data + replaced files will be backed up at $TESTTMP/modern/.hg/upgradebackup.* (glob) + replacing store... + store replacement complete; repository was inconsistent for *s (glob) + finalizing requirements file and making repository readable again + removing temporary repository $TESTTMP/modern/.hg/upgrade.* (glob) + copy of old repository backed up at $TESTTMP/modern/.hg/upgradebackup.* (glob) + the old repository will not be deleted; remove it to free up disk space once the upgraded repository is verified + +Upgrading a repository to generaldelta works + + $ hg --config format.usegeneraldelta=false init upgradegd + $ cd upgradegd + $ touch f0 + $ hg -q commit -A -m initial + $ touch f1 + $ hg -q commit -A -m 'add f1' + $ hg -q up -r 0 + $ touch f2 + $ hg -q commit -A -m 'add f2' + + $ hg debugupgraderepo --run + upgrade will perform the following actions: + + requirements + preserved: dotencode, fncache, revlogv1, store + added: generaldelta + + generaldelta + repository storage will be able to create optimal deltas; new repository data will be smaller and read times should decrease; interacting with other repositories using this storage model should require less network and CPU resources, making "hg push" and "hg pull" faster + + beginning upgrade... + repository locked and read-only + creating temporary repository to stage migrated data: $TESTTMP/upgradegd/.hg/upgrade.* (glob) + (it is safe to interrupt this process any time before data migration completes) + migrating 9 total revisions (3 in filelogs, 3 in manifests, 3 in changelog) + migrating 341 bytes in store; 401 bytes tracked data + migrating 3 filelogs containing 3 revisions (0 bytes in store; 0 bytes tracked data) + finished migrating 3 filelog revisions across 3 filelogs; change in size: 0 bytes + migrating 1 manifests containing 3 revisions (157 bytes in store; 220 bytes tracked data) + finished migrating 3 manifest revisions across 1 manifests; change in size: 0 bytes + migrating changelog containing 3 revisions (184 bytes in store; 181 bytes tracked data) + finished migrating 3 changelog revisions; change in size: 0 bytes + finished migrating 9 total revisions; total change in store size: 0 bytes + copying phaseroots + data fully migrated to temporary repository + marking source repository as being upgraded; clients will be unable to read from repository + starting in-place swap of repository data + replaced files will be backed up at $TESTTMP/upgradegd/.hg/upgradebackup.* (glob) + replacing store... + store replacement complete; repository was inconsistent for *s (glob) + finalizing requirements file and making repository readable again + removing temporary repository $TESTTMP/upgradegd/.hg/upgrade.* (glob) + copy of old repository backed up at $TESTTMP/upgradegd/.hg/upgradebackup.* (glob) + the old repository will not be deleted; remove it to free up disk space once the upgraded repository is verified + +Original requirements backed up + + $ cat .hg/upgradebackup.*/requires + dotencode + fncache + revlogv1 + store + +generaldelta added to original requirements files + + $ cat .hg/requires + dotencode + fncache + generaldelta + revlogv1 + store + +store directory has files we expect + + $ ls .hg/store + 00changelog.i + 00manifest.i + data + fncache + phaseroots + undo + undo.backupfiles + undo.phaseroots + +manifest should be generaldelta + + $ hg debugrevlog -m | grep flags + flags : inline, generaldelta + +verify should be happy + + $ hg verify + checking changesets + checking manifests + crosschecking files in changesets and manifests + checking files + 3 files, 3 changesets, 3 total revisions + +old store should be backed up + + $ ls .hg/upgradebackup.*/store + 00changelog.i + 00manifest.i + data + fncache + phaseroots + undo + undo.backup.fncache + undo.backupfiles + undo.phaseroots + + $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-verify.t mercurial-4.1~rc~trusty1/tests/test-verify.t --- mercurial-4.0.2~trusty1/tests/test-verify.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-verify.t 2017-01-18 16:54:51.000000000 +0000 @@ -69,10 +69,10 @@ $ cd missing-entries $ echo 0 > file $ hg ci -Aqm0 - $ cp -r .hg/store .hg/store-partial + $ cp -R .hg/store .hg/store-partial $ echo 1 > file $ hg ci -Aqm1 - $ cp -r .hg/store .hg/store-full + $ cp -R .hg/store .hg/store-full Entire changelog missing @@ -84,7 +84,7 @@ 3 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Entire manifest log missing @@ -94,7 +94,7 @@ 1 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Entire filelog missing @@ -109,7 +109,7 @@ 3 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Entire changelog and manifest log missing @@ -118,7 +118,7 @@ $ hg verify -q warning: orphan revlog 'data/file.i' 1 warnings encountered! - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Entire changelog and filelog missing @@ -137,7 +137,7 @@ 6 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Entire manifest log and filelog missing @@ -152,7 +152,7 @@ 2 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Changelog missing entry @@ -165,7 +165,7 @@ 1 warnings encountered! 3 integrity errors encountered! [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Manifest log missing entry @@ -176,7 +176,7 @@ 2 integrity errors encountered! (first damaged changeset appears to be 1) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Filelog missing entry @@ -186,7 +186,7 @@ 1 integrity errors encountered! (first damaged changeset appears to be 1) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Changelog and manifest log missing entry @@ -199,7 +199,7 @@ 1 warnings encountered! 2 integrity errors encountered! [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Changelog and filelog missing entry @@ -211,7 +211,7 @@ file@?: manifest refers to unknown revision c10f2164107d 3 integrity errors encountered! [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Manifest and filelog missing entry @@ -222,7 +222,7 @@ 1 integrity errors encountered! (first damaged changeset appears to be 1) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Corrupt changelog base node to cause failure to read revision @@ -238,7 +238,7 @@ 4 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Corrupt manifest log base node to cause failure to read revision @@ -250,7 +250,7 @@ 2 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store Corrupt filelog base node to cause failure to read revision @@ -261,7 +261,7 @@ 1 integrity errors encountered! (first damaged changeset appears to be 0) [1] - $ cp -r .hg/store-full/. .hg/store + $ cp -R .hg/store-full/. .hg/store $ cd .. diff -Nru mercurial-4.0.2~trusty1/tests/test-walkrepo.py mercurial-4.1~rc~trusty1/tests/test-walkrepo.py --- mercurial-4.0.2~trusty1/tests/test-walkrepo.py 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-walkrepo.py 2017-01-18 16:54:51.000000000 +0000 @@ -16,7 +16,7 @@ walkrepos = scmutil.walkrepos checklink = util.checklink -u = uimod.ui() +u = uimod.ui.load() sym = checklink('.') hg.repository(u, 'top1', create=1) diff -Nru mercurial-4.0.2~trusty1/tests/test-wireproto.t mercurial-4.1~rc~trusty1/tests/test-wireproto.t --- mercurial-4.0.2~trusty1/tests/test-wireproto.t 2016-12-31 08:19:09.000000000 +0000 +++ mercurial-4.1~rc~trusty1/tests/test-wireproto.t 2017-01-18 16:54:51.000000000 +0000 @@ -94,23 +94,23 @@ * - - [*] "POST /?cmd=debugwireargs HTTP/1.1" 200 - x-hgargs-post:1033* (glob) * - - [*] "POST /?cmd=debugwireargs HTTP/1.1" 200 - x-hgargs-post:1033* (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=quatre&one=un&three=trois&two=deux (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=quatre&one=un&three=trois&two=deux (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=quatre&one=un&three=trois&two=deux x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=quatre&one=un&three=trois&two=deux x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=qu++atre&one=+un&three=trois+&two=deux (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=qu++atre&one=+un&three=trois+&two=deux (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=qu++atre&one=+un&three=trois+&two=deux x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=qu++atre&one=+un&three=trois+&two=deux x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=vier&one=eins&two=zwei (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=vier&one=eins&two=zwei (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=vier&one=eins&two=zwei x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=vier&one=eins&two=zwei x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:one=eins&two=zwei x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=onethousandcharactersxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&one x-hgarg-2:=un&three=trois&two=deux (glob) - * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=onethousandcharactersxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&one x-hgarg-2:=un&three=trois&two=deux (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=onethousandcharactersxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&one x-hgarg-2:=un&three=trois&two=deux x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs HTTP/1.1" 200 - x-hgarg-1:four=onethousandcharactersxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&one x-hgarg-2:=un&three=trois&two=deux x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) HTTP without the httpheader capability: @@ -133,17 +133,17 @@ $ cat error2.log $ cat access2.log * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&four=quatre&one=un&three=trois&two=deux HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&four=quatre&one=un&three=trois&two=deux HTTP/1.1" 200 - (glob) + * - - [*] "GET /?cmd=debugwireargs&four=quatre&one=un&three=trois&two=deux HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs&four=quatre&one=un&three=trois&two=deux HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&four=vier&one=eins&two=zwei HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&four=vier&one=eins&two=zwei HTTP/1.1" 200 - (glob) + * - - [*] "GET /?cmd=debugwireargs&four=vier&one=eins&two=zwei HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs&four=vier&one=eins&two=zwei HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - (glob) + * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) * - - [*] "GET /?cmd=capabilities HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - (glob) - * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - (glob) + * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) + * - - [*] "GET /?cmd=debugwireargs&one=eins&two=zwei HTTP/1.1" 200 - x-hgproto-1:0.1 0.2 comp=*zlib,none,bzip2 (glob) SSH (try to exercise the ssh functionality with a dummy script):