diff -Nru s3ql-2.22+dfsg/Changes.txt s3ql-2.23+dfsg/Changes.txt --- s3ql-2.22+dfsg/Changes.txt 2017-06-23 16:36:18.000000000 +0000 +++ s3ql-2.23+dfsg/Changes.txt 2017-08-18 18:11:06.000000000 +0000 @@ -1,3 +1,16 @@ +2017-08-18, S3QL 2.23 + + * Accessing the us-east-1 S3 storage region is now working again. + + * Retry for some additional SSL Errors. Unfortunately we're playing + whack-a-mole here. + + * The swift backend now dynamically detects the features supported + by the backend server. + + * The swift backend now supports optimized object copy and bulk + deletion (if supported by the server). + 2017-06-23, S3QL 2.22 * The storage URL for Amazon S3 buckets has changed. It now includes diff -Nru s3ql-2.22+dfsg/contrib/expire_backups.1 s3ql-2.23+dfsg/contrib/expire_backups.1 --- s3ql-2.22+dfsg/contrib/expire_backups.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/contrib/expire_backups.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "EXPIRE_BACKUPS" "1" "June 23, 2017" "2.22" "S3QL" +.TH "EXPIRE_BACKUPS" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME expire_backups \- Intelligently expire old backups . @@ -105,7 +105,7 @@ .sp When storing your backups on an S3QL file system, you probably want to specify the \fB\-\-use\-s3qlrm\fP option as well. This tells -\fBexpire_backups\fP to use the \fIs3qlrm\fP command to +\fBexpire_backups\fP to use the s3qlrm command to delete directories. .sp \fBexpire_backups\fP uses a "state file" to keep track which diff -Nru s3ql-2.22+dfsg/contrib/pcp.1 s3ql-2.23+dfsg/contrib/pcp.1 --- s3ql-2.22+dfsg/contrib/pcp.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/contrib/pcp.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PCP" "1" "June 23, 2017" "2.22" "S3QL" +.TH "PCP" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME pcp \- Recursive, parallel copy of directory trees . diff -Nru s3ql-2.22+dfsg/debian/changelog s3ql-2.23+dfsg/debian/changelog --- s3ql-2.22+dfsg/debian/changelog 2017-07-24 20:47:28.000000000 +0000 +++ s3ql-2.23+dfsg/debian/changelog 2017-08-19 12:38:51.000000000 +0000 @@ -1,8 +1,8 @@ -s3ql (2.22+dfsg-1build1) artful; urgency=medium +s3ql (2.23+dfsg-1) unstable; urgency=medium - * No-change build for python3.6. + * New upstream release. - -- Matthias Klose Mon, 24 Jul 2017 22:47:28 +0200 + -- Nikolaus Rath Sat, 19 Aug 2017 14:38:51 +0200 s3ql (2.22+dfsg-1) unstable; urgency=medium diff -Nru s3ql-2.22+dfsg/debian/.git-dpm s3ql-2.23+dfsg/debian/.git-dpm --- s3ql-2.22+dfsg/debian/.git-dpm 2017-06-23 16:53:02.000000000 +0000 +++ s3ql-2.23+dfsg/debian/.git-dpm 2017-08-19 12:38:42.000000000 +0000 @@ -1,11 +1,11 @@ # see git-dpm(1) from git-dpm package -748cbaafa526bc1e065232b7fbd80b37b92efc78 -748cbaafa526bc1e065232b7fbd80b37b92efc78 -6491fdcd035d05feee3f5e7d8f3d46aec37fb8ec -6491fdcd035d05feee3f5e7d8f3d46aec37fb8ec -s3ql_2.22+dfsg.orig.tar.gz -60e31d4d4d24c53598ad03d6eb65e93f5914fd41 -650427 +379330c978c2e11e92dd3955a4e67fd96c2433e8 +379330c978c2e11e92dd3955a4e67fd96c2433e8 +a5eebe2087e42d511efd3f03738a3d5688684b5c +a5eebe2087e42d511efd3f03738a3d5688684b5c +s3ql_2.23+dfsg.orig.tar.gz +adb683a0268fd0843f3ce39d6d2e1a524386090b +674783 debianTag="debian/%e%v" patchedTag="patched/%e%v" upstreamTag="upstream/%e%u" diff -Nru s3ql-2.22+dfsg/debian/patches/0005-Expect-test_verify-to-fail-during-build.patch s3ql-2.23+dfsg/debian/patches/0005-Expect-test_verify-to-fail-during-build.patch --- s3ql-2.22+dfsg/debian/patches/0005-Expect-test_verify-to-fail-during-build.patch 2017-06-23 16:53:02.000000000 +0000 +++ s3ql-2.23+dfsg/debian/patches/0005-Expect-test_verify-to-fail-during-build.patch 2017-08-19 12:38:42.000000000 +0000 @@ -1,4 +1,4 @@ -From 748cbaafa526bc1e065232b7fbd80b37b92efc78 Mon Sep 17 00:00:00 2001 +From 379330c978c2e11e92dd3955a4e67fd96c2433e8 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 17 Jan 2017 20:34:23 -0800 Subject: Expect test_verify() to fail during build. diff -Nru s3ql-2.22+dfsg/debian/patches/clock-granularity.diff s3ql-2.23+dfsg/debian/patches/clock-granularity.diff --- s3ql-2.22+dfsg/debian/patches/clock-granularity.diff 2017-06-23 16:53:02.000000000 +0000 +++ s3ql-2.23+dfsg/debian/patches/clock-granularity.diff 2017-08-19 12:38:42.000000000 +0000 @@ -1,4 +1,4 @@ -From fe88bb4bfea241b2095ef9eb0410680647546722 Mon Sep 17 00:00:00 2001 +From 57d8af1f2bb712fc6331af5caf8ce8cf80529cf4 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Estimate system clock granularity more conservatively diff -Nru s3ql-2.22+dfsg/debian/patches/ignore_cython_warnings.diff s3ql-2.23+dfsg/debian/patches/ignore_cython_warnings.diff --- s3ql-2.22+dfsg/debian/patches/ignore_cython_warnings.diff 2017-06-23 16:53:02.000000000 +0000 +++ s3ql-2.23+dfsg/debian/patches/ignore_cython_warnings.diff 2017-08-19 12:38:42.000000000 +0000 @@ -1,4 +1,4 @@ -From 55caf8c017ece10a29a9b3afc1b1f3a101d0ba22 Mon Sep 17 00:00:00 2001 +From 3e796010faae9879f40bc7c4037f2a63f9e48761 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Ignore compiler warnings due to old Cython version diff -Nru s3ql-2.22+dfsg/debian/patches/proc_mount.diff s3ql-2.23+dfsg/debian/patches/proc_mount.diff --- s3ql-2.22+dfsg/debian/patches/proc_mount.diff 2017-06-23 16:53:02.000000000 +0000 +++ s3ql-2.23+dfsg/debian/patches/proc_mount.diff 2017-08-19 12:38:42.000000000 +0000 @@ -1,4 +1,4 @@ -From bc71ed52bd12a67c3681df12de8d2316052ba792 Mon Sep 17 00:00:00 2001 +From d10b189aa9d3baad09a2be98c3156f4f06da84c6 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Tue, 23 Feb 2016 15:44:51 -0800 Subject: Skip tests requiring /proc diff -Nru s3ql-2.22+dfsg/debian/patches/show_pdflatex_output.diff s3ql-2.23+dfsg/debian/patches/show_pdflatex_output.diff --- s3ql-2.22+dfsg/debian/patches/show_pdflatex_output.diff 2017-06-23 16:53:02.000000000 +0000 +++ s3ql-2.23+dfsg/debian/patches/show_pdflatex_output.diff 2017-08-19 12:38:42.000000000 +0000 @@ -1,4 +1,4 @@ -From bc59be83cb228a5dec73500c02677a1c6898a73e Mon Sep 17 00:00:00 2001 +From 72643423b1493ae16449e14dc39a4a08aa689a73 Mon Sep 17 00:00:00 2001 From: Nikolaus Rath Date: Mon, 27 Jun 2016 11:25:34 -0700 Subject: Don't hide pdflatex output. diff -Nru s3ql-2.22+dfsg/doc/latex/Makefile s3ql-2.23+dfsg/doc/latex/Makefile --- s3ql-2.22+dfsg/doc/latex/Makefile 2014-05-03 03:50:10.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/Makefile 2016-11-23 14:06:39.000000000 +0000 @@ -8,6 +8,12 @@ ARCHIVEPRREFIX = # Additional LaTeX options LATEXOPTS = +# format: pdf or dvi +FMT = pdf + +LATEX = latex +PDFLATEX = pdflatex +MAKEINDEX = makeindex all: $(ALLPDF) all-pdf: $(ALLPDF) @@ -37,30 +43,36 @@ tar cf $(ARCHIVEPREFIX)docs-$(FMT).tar $(ARCHIVEPREFIX)docs-$(FMT) rm -r $(ARCHIVEPREFIX)docs-$(FMT) +gz: tar + gzip -9 < $(ARCHIVEPREFIX)docs-$(FMT).tar > $(ARCHIVEPREFIX)docs-$(FMT).tar.gz + bz2: tar bzip2 -9 -k $(ARCHIVEPREFIX)docs-$(FMT).tar +xz: tar + xz -9 -k $(ARCHIVEPREFIX)docs-$(FMT).tar + # The number of LaTeX runs is quite conservative, but I don't expect it # to get run often, so the little extra time won't hurt. %.dvi: %.tex - latex $(LATEXOPTS) '$<' - latex $(LATEXOPTS) '$<' - latex $(LATEXOPTS) '$<' - -makeindex -s python.ist '$(basename $<).idx' - latex $(LATEXOPTS) '$<' - latex $(LATEXOPTS) '$<' + $(LATEX) $(LATEXOPTS) '$<' + $(LATEX) $(LATEXOPTS) '$<' + $(LATEX) $(LATEXOPTS) '$<' + -$(MAKEINDEX) -s python.ist '$(basename $<).idx' + $(LATEX) $(LATEXOPTS) '$<' + $(LATEX) $(LATEXOPTS) '$<' %.pdf: %.tex - pdflatex $(LATEXOPTS) '$<' - pdflatex $(LATEXOPTS) '$<' - pdflatex $(LATEXOPTS) '$<' - -makeindex -s python.ist '$(basename $<).idx' - pdflatex $(LATEXOPTS) '$<' - pdflatex $(LATEXOPTS) '$<' + $(PDFLATEX) $(LATEXOPTS) '$<' + $(PDFLATEX) $(LATEXOPTS) '$<' + $(PDFLATEX) $(LATEXOPTS) '$<' + -$(MAKEINDEX) -s python.ist '$(basename $<).idx' + $(PDFLATEX) $(LATEXOPTS) '$<' + $(PDFLATEX) $(LATEXOPTS) '$<' clean: - rm -f *.dvi *.log *.ind *.aux *.toc *.syn *.idx *.out *.ilg *.pla + rm -f *.log *.ind *.aux *.toc *.syn *.idx *.out *.ilg *.pla *.ps *.tar *.tar.gz *.tar.bz2 *.tar.xz $(ALLPDF) $(ALLDVI) -.PHONY: all all-pdf all-dvi all-ps clean +.PHONY: all all-pdf all-dvi all-ps clean zip tar gz bz2 xz .PHONY: all-pdf-ja diff -Nru s3ql-2.22+dfsg/doc/latex/manual.aux s3ql-2.23+dfsg/doc/latex/manual.aux --- s3ql-2.22+dfsg/doc/latex/manual.aux 2017-06-23 16:36:56.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/manual.aux 2017-08-18 18:11:57.000000000 +0000 @@ -24,8 +24,9 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {1}About S3QL}{1}{chapter.1}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{about:s3ql-user-s-guide}{{1}{1}{About S3QL}{chapter.1}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{about:about-s3ql}{{1}{1}{About S3QL}{chapter.1}{}} +\newlabel{about:s3ql-user-s-guide}{{1}{1}{About S3QL}{chapter.1}{}} \newlabel{about::doc}{{1}{1}{About S3QL}{chapter.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {1.1}Features}{1}{section.1.1}} \newlabel{about:features}{{1.1}{1}{Features}{section.1.1}{}} @@ -39,9 +40,10 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {2}Installation}{3}{chapter.2}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{installation:github}{{2}{3}{Installation}{chapter.2}{}} -\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} \newlabel{installation::doc}{{2}{3}{Installation}{chapter.2}{}} +\newlabel{installation:installation}{{2}{3}{Installation}{chapter.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.1}Dependencies}{3}{section.2.1}} \newlabel{installation:dependencies}{{2.1}{3}{Dependencies}{section.2.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {2.2}Installing S3QL}{4}{section.2.2}} @@ -54,18 +56,19 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {3}Storage Backends}{7}{chapter.3}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{backends:sphinx}{{3}{7}{Storage Backends}{chapter.3}{}} +\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:id1}{{3}{7}{Storage Backends}{chapter.3}{}} \newlabel{backends:storage-backends}{{3}{7}{Storage Backends}{chapter.3}{}} -\newlabel{backends::doc}{{3}{7}{Storage Backends}{chapter.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1}} \newlabel{backends:google-storage}{{3.1}{7}{Google Storage}{section.3.1}{}} \newlabel{backends:cmdoption-gs_backend-arg-no-ssl}{{3.1}{7}{Google Storage}{section*.3}{}} \newlabel{backends:cmdoption-gs_backend-arg-ssl-ca-path}{{3.1}{8}{Google Storage}{section*.4}{}} \newlabel{backends:cmdoption-gs_backend-arg-tcp-timeout}{{3.1}{8}{Google Storage}{section*.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2}} -\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:google-storage-manager}{{3.2}{8}{Amazon S3}{section.3.2}{}} +\newlabel{backends:amazon-s3}{{3.2}{8}{Amazon S3}{section.3.2}{}} \newlabel{backends:cmdoption-s3_backend-arg-no-ssl}{{3.2}{8}{Amazon S3}{section*.6}{}} \newlabel{backends:cmdoption-s3_backend-arg-ssl-ca-path}{{3.2}{8}{Amazon S3}{section*.7}{}} \newlabel{backends:cmdoption-s3_backend-arg-tcp-timeout}{{3.2}{8}{Amazon S3}{section*.8}{}} @@ -79,27 +82,28 @@ \newlabel{backends:cmdoption-swift_backend-arg-ssl-ca-path}{{3.3}{9}{OpenStack/Swift}{section*.13}{}} \newlabel{backends:cmdoption-swift_backend-arg-tcp-timeout}{{3.3}{9}{OpenStack/Swift}{section*.14}{}} \newlabel{backends:cmdoption-swift_backend-arg-disable-expect100}{{3.3}{9}{OpenStack/Swift}{section*.15}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{9}{section.3.4}} -\newlabel{backends:rackspace-cloudfiles}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} -\newlabel{backends:swift}{{3.4}{9}{Rackspace CloudFiles}{section.3.4}{}} +\newlabel{backends:cmdoption-swift_backend-arg-no-feature-detection}{{3.3}{9}{OpenStack/Swift}{section*.16}{}} +\@writefile{toc}{\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4}} +\newlabel{backends:rackspace-cloudfiles}{{3.4}{10}{Rackspace CloudFiles}{section.3.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5}} \newlabel{backends:rackspace}{{3.5}{10}{S3 compatible}{section.3.5}{}} \newlabel{backends:s3-compatible}{{3.5}{10}{S3 compatible}{section.3.5}{}} -\newlabel{backends:cmdoption-s3c_backend-arg-no-ssl}{{3.5}{10}{S3 compatible}{section*.16}{}} -\newlabel{backends:cmdoption-s3c_backend-arg-ssl-ca-path}{{3.5}{10}{S3 compatible}{section*.17}{}} -\newlabel{backends:cmdoption-s3c_backend-arg-tcp-timeout}{{3.5}{10}{S3 compatible}{section*.18}{}} -\newlabel{backends:cmdoption-s3c_backend-arg-disable-expect100}{{3.5}{10}{S3 compatible}{section*.19}{}} -\newlabel{backends:cmdoption-s3c_backend-arg-dumb-copy}{{3.5}{10}{S3 compatible}{section*.20}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-no-ssl}{{3.5}{10}{S3 compatible}{section*.17}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-ssl-ca-path}{{3.5}{10}{S3 compatible}{section*.18}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-tcp-timeout}{{3.5}{10}{S3 compatible}{section*.19}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-disable-expect100}{{3.5}{10}{S3 compatible}{section*.20}{}} +\newlabel{backends:cmdoption-s3c_backend-arg-dumb-copy}{{3.5}{10}{S3 compatible}{section*.21}{}} \@writefile{toc}{\contentsline {section}{\numberline {3.6}Local}{11}{section.3.6}} -\newlabel{backends:id6}{{3.6}{11}{Local}{section.3.6}{}} \newlabel{backends:local}{{3.6}{11}{Local}{section.3.6}{}} +\newlabel{backends:id6}{{3.6}{11}{Local}{section.3.6}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{durability:durability}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} -\newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:sshfs}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \newlabel{durability::doc}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} +\newlabel{durability:important-rules-to-avoid-losing-data}{{4}{13}{Important Rules to Avoid Losing Data}{chapter.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.1}Rules in a Nutshell}{13}{section.4.1}} \newlabel{durability:rules-in-a-nutshell}{{4.1}{13}{Rules in a Nutshell}{section.4.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {4.2}Consistency Window List}{14}{section.4.2}} @@ -112,11 +116,13 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {5}File System Creation}{17}{chapter.5}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{mkfs:file-system-creation}{{5}{17}{File System Creation}{chapter.5}{}} \newlabel{mkfs::doc}{{5}{17}{File System Creation}{chapter.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {6}Managing File Systems}{19}{chapter.6}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{adm:managing-file-systems}{{6}{19}{Managing File Systems}{chapter.6}{}} \newlabel{adm::doc}{{6}{19}{Managing File Systems}{chapter.6}{}} \@writefile{toc}{\contentsline {section}{\numberline {6.1}Changing the Passphrase}{19}{section.6.1}} @@ -130,53 +136,57 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{mount:mounting}{{7}{21}{Mounting}{chapter.7}{}} \newlabel{mount::doc}{{7}{21}{Mounting}{chapter.7}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1}} \newlabel{mount:permission-checking}{{7.1}{22}{Permission Checking}{section.7.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2}} \newlabel{mount:compression-algorithms}{{7.2}{22}{Compression Algorithms}{section.7.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.3}Notes about Caching}{22}{section.7.3}} -\newlabel{mount:notes-about-caching}{{7.3}{22}{Notes about Caching}{section.7.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.3}Notes about Caching}{23}{section.7.3}} +\newlabel{mount:notes-about-caching}{{7.3}{23}{Notes about Caching}{section.7.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1}} \newlabel{mount:maximum-number-of-cache-entries}{{7.3.1}{23}{Maximum Number of Cache Entries}{subsection.7.3.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2}} \newlabel{mount:cache-flushing-and-expiration}{{7.3.2}{23}{Cache Flushing and Expiration}{subsection.7.3.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4}} \newlabel{mount:failure-modes}{{7.4}{23}{Failure Modes}{section.7.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {7.5}Automatic Mounting}{23}{section.7.5}} -\newlabel{mount:automatic-mounting}{{7.5}{23}{Automatic Mounting}{section.7.5}{}} -\newlabel{mount:logcheck}{{7.5}{23}{Automatic Mounting}{section.7.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {7.5}Automatic Mounting}{24}{section.7.5}} +\newlabel{mount:automatic-mounting}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} +\newlabel{mount:logcheck}{{7.5}{24}{Automatic Mounting}{section.7.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{special:advanced-s3ql-features}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{special::doc}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} +\newlabel{special:advanced-s3ql-features}{{8}{25}{Advanced S3QL Features}{chapter.8}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1}} \newlabel{special:snapshotting-and-copy-on-write}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} \newlabel{special:s3qlcp}{{8.1}{25}{Snapshotting and Copy-on-Write}{section.8.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1}} \newlabel{special:snapshotting-vs-hardlinking}{{8.1.1}{25}{Snapshotting vs Hardlinking}{subsection.8.1.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.2}Getting Statistics}{26}{section.8.2}} -\newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \newlabel{special:s3qlstat}{{8.2}{26}{Getting Statistics}{section.8.2}{}} +\newlabel{special:getting-statistics}{{8.2}{26}{Getting Statistics}{section.8.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.3}Immutable Trees}{26}{section.8.3}} -\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \newlabel{special:immutable-trees}{{8.3}{26}{Immutable Trees}{section.8.3}{}} +\newlabel{special:s3qllock}{{8.3}{26}{Immutable Trees}{section.8.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.4}Fast Recursive Removal}{27}{section.8.4}} -\newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \newlabel{special:fast-recursive-removal}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} +\newlabel{special:s3qlrm}{{8.4}{27}{Fast Recursive Removal}{section.8.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {8.5}Runtime Configuration}{27}{section.8.5}} \newlabel{special:s3qlctrl}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} \newlabel{special:runtime-configuration}{{8.5}{27}{Runtime Configuration}{section.8.5}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {9}Unmounting}{29}{chapter.9}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{umount::doc}{{9}{29}{Unmounting}{chapter.9}{}} +\newlabel{umount:unmounting}{{9}{29}{Unmounting}{chapter.9}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {10}Checking for Errors}{31}{chapter.10}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{fsck:checking-for-errors}{{10}{31}{Checking for Errors}{chapter.10}{}} \newlabel{fsck::doc}{{10}{31}{Checking for Errors}{chapter.10}{}} \@writefile{toc}{\contentsline {section}{\numberline {10.1}Checking and repairing internal file system errors}{31}{section.10.1}} @@ -187,12 +197,14 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {11}Storing Authentication Information}{33}{chapter.11}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{authinfo:authinfo}{{11}{33}{Storing Authentication Information}{chapter.11}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{authinfo:storing-authentication-information}{{11}{33}{Storing Authentication Information}{chapter.11}{}} +\newlabel{authinfo:authinfo}{{11}{33}{Storing Authentication Information}{chapter.11}{}} \newlabel{authinfo::doc}{{11}{33}{Storing Authentication Information}{chapter.11}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {12}Contributed Programs}{35}{chapter.12}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{contrib:contributed-programs}{{12}{35}{Contributed Programs}{chapter.12}{}} \newlabel{contrib::doc}{{12}{35}{Contributed Programs}{chapter.12}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.1}benchmark.py}{35}{section.12.1}} @@ -200,23 +212,24 @@ \@writefile{toc}{\contentsline {section}{\numberline {12.2}clone\_fs.py}{35}{section.12.2}} \newlabel{contrib:clone-fs-py}{{12.2}{35}{clone\_fs.py}{section.12.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.3}pcp.py}{35}{section.12.3}} -\newlabel{contrib:pcp}{{12.3}{35}{pcp.py}{section.12.3}{}} \newlabel{contrib:pcp-py}{{12.3}{35}{pcp.py}{section.12.3}{}} +\newlabel{contrib:pcp}{{12.3}{35}{pcp.py}{section.12.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.4}s3ql\_backup.sh}{35}{section.12.4}} \newlabel{contrib:s3ql-backup-sh}{{12.4}{35}{s3ql\_backup.sh}{section.12.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.5}expire\_backups.py}{36}{section.12.5}} \newlabel{contrib:expire-backups-py}{{12.5}{36}{expire\_backups.py}{section.12.5}{}} \@writefile{toc}{\contentsline {section}{\numberline {12.6}remove\_objects.py}{37}{section.12.6}} -\newlabel{contrib:remove-objects}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} \newlabel{contrib:remove-objects-py}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} +\newlabel{contrib:remove-objects}{{12.6}{37}{remove\_objects.py}{section.12.6}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {13}Tips \& Tricks}{39}{chapter.13}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{tips:tips-tricks}{{13}{39}{Tips \& Tricks}{chapter.13}{}} \newlabel{tips::doc}{{13}{39}{Tips \& Tricks}{chapter.13}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.1}SSH Backend}{39}{section.13.1}} -\newlabel{tips:ssh-tipp}{{13.1}{39}{SSH Backend}{section.13.1}{}} \newlabel{tips:ssh-backend}{{13.1}{39}{SSH Backend}{section.13.1}{}} +\newlabel{tips:ssh-tipp}{{13.1}{39}{SSH Backend}{section.13.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.2}Permanently mounted backup file system}{39}{section.13.2}} \newlabel{tips:permanently-mounted-backup-file-system}{{13.2}{39}{Permanently mounted backup file system}{section.13.2}{}} \@writefile{toc}{\contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3}} @@ -225,16 +238,18 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{issues:known-issues}{{14}{41}{Known Issues}{chapter.14}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{issues::doc}{{14}{41}{Known Issues}{chapter.14}{}} +\newlabel{issues:known-issues}{{14}{41}{Known Issues}{chapter.14}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {15}Manpages}{43}{chapter.15}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} -\newlabel{man/index:manpages}{{15}{43}{Manpages}{chapter.15}{}} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{man/index::doc}{{15}{43}{Manpages}{chapter.15}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.1}The \textbf {mkfs.s3ql} command}{43}{section.15.1}} -\newlabel{man/mkfs:the-command-command}{{15.1}{43}{The \textbf {mkfs.s3ql} command}{section.15.1}{}} -\newlabel{man/mkfs::doc}{{15.1}{43}{The \textbf {mkfs.s3ql} command}{section.15.1}{}} +\newlabel{man/index:manpages}{{15}{43}{Manpages}{chapter.15}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.1}The \textbf {\texttt {mkfs.s3ql}} command}{43}{section.15.1}} +\newlabel{man/mkfs:the-command-command}{{15.1}{43}{The \textbf {\texttt {mkfs.s3ql}} command}{section.15.1}{}} +\newlabel{man/mkfs::doc}{{15.1}{43}{The \textbf {\texttt {mkfs.s3ql}} command}{section.15.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.1.1}Synopsis}{43}{subsection.15.1.1}} \newlabel{man/mkfs:synopsis}{{15.1.1}{43}{Synopsis}{subsection.15.1.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.1.2}Description}{43}{subsection.15.1.2}} @@ -245,9 +260,9 @@ \newlabel{man/mkfs:exit-codes}{{15.1.4}{44}{Exit Codes}{subsection.15.1.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.1.5}See Also}{44}{subsection.15.1.5}} \newlabel{man/mkfs:see-also}{{15.1.5}{44}{See Also}{subsection.15.1.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.2}The \textbf {s3qladm} command}{44}{section.15.2}} -\newlabel{man/adm:the-command-command}{{15.2}{44}{The \textbf {s3qladm} command}{section.15.2}{}} -\newlabel{man/adm::doc}{{15.2}{44}{The \textbf {s3qladm} command}{section.15.2}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.2}The \textbf {\texttt {s3qladm}} command}{44}{section.15.2}} +\newlabel{man/adm:the-command-command}{{15.2}{44}{The \textbf {\texttt {s3qladm}} command}{section.15.2}{}} +\newlabel{man/adm::doc}{{15.2}{44}{The \textbf {\texttt {s3qladm}} command}{section.15.2}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.2.1}Synopsis}{44}{subsection.15.2.1}} \newlabel{man/adm:synopsis}{{15.2.1}{44}{Synopsis}{subsection.15.2.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.2.2}Description}{45}{subsection.15.2.2}} @@ -260,9 +275,9 @@ \newlabel{man/adm:exit-codes}{{15.2.5}{45}{Exit Codes}{subsection.15.2.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.2.6}See Also}{46}{subsection.15.2.6}} \newlabel{man/adm:see-also}{{15.2.6}{46}{See Also}{subsection.15.2.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.3}The \textbf {mount.s3ql} command}{46}{section.15.3}} -\newlabel{man/mount:the-command-command}{{15.3}{46}{The \textbf {mount.s3ql} command}{section.15.3}{}} -\newlabel{man/mount::doc}{{15.3}{46}{The \textbf {mount.s3ql} command}{section.15.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.3}The \textbf {\texttt {mount.s3ql}} command}{46}{section.15.3}} +\newlabel{man/mount:the-command-command}{{15.3}{46}{The \textbf {\texttt {mount.s3ql}} command}{section.15.3}{}} +\newlabel{man/mount::doc}{{15.3}{46}{The \textbf {\texttt {mount.s3ql}} command}{section.15.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.3.1}Synopsis}{46}{subsection.15.3.1}} \newlabel{man/mount:synopsis}{{15.3.1}{46}{Synopsis}{subsection.15.3.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.3.2}Description}{46}{subsection.15.3.2}} @@ -273,9 +288,9 @@ \newlabel{man/mount:exit-codes}{{15.3.4}{47}{Exit Codes}{subsection.15.3.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.3.5}See Also}{48}{subsection.15.3.5}} \newlabel{man/mount:see-also}{{15.3.5}{48}{See Also}{subsection.15.3.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.4}The \textbf {s3qlstat} command}{48}{section.15.4}} -\newlabel{man/stat:the-command-command}{{15.4}{48}{The \textbf {s3qlstat} command}{section.15.4}{}} -\newlabel{man/stat::doc}{{15.4}{48}{The \textbf {s3qlstat} command}{section.15.4}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.4}The \textbf {\texttt {s3qlstat}} command}{48}{section.15.4}} +\newlabel{man/stat:the-command-command}{{15.4}{48}{The \textbf {\texttt {s3qlstat}} command}{section.15.4}{}} +\newlabel{man/stat::doc}{{15.4}{48}{The \textbf {\texttt {s3qlstat}} command}{section.15.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.4.1}Synopsis}{48}{subsection.15.4.1}} \newlabel{man/stat:synopsis}{{15.4.1}{48}{Synopsis}{subsection.15.4.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.4.2}Description}{48}{subsection.15.4.2}} @@ -286,9 +301,9 @@ \newlabel{man/stat:exit-codes}{{15.4.4}{49}{Exit Codes}{subsection.15.4.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.4.5}See Also}{49}{subsection.15.4.5}} \newlabel{man/stat:see-also}{{15.4.5}{49}{See Also}{subsection.15.4.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.5}The \textbf {s3qlctrl} command}{49}{section.15.5}} -\newlabel{man/ctrl:the-command-command}{{15.5}{49}{The \textbf {s3qlctrl} command}{section.15.5}{}} -\newlabel{man/ctrl::doc}{{15.5}{49}{The \textbf {s3qlctrl} command}{section.15.5}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.5}The \textbf {\texttt {s3qlctrl}} command}{49}{section.15.5}} +\newlabel{man/ctrl:the-command-command}{{15.5}{49}{The \textbf {\texttt {s3qlctrl}} command}{section.15.5}{}} +\newlabel{man/ctrl::doc}{{15.5}{49}{The \textbf {\texttt {s3qlctrl}} command}{section.15.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.5.1}Synopsis}{49}{subsection.15.5.1}} \newlabel{man/ctrl:synopsis}{{15.5.1}{49}{Synopsis}{subsection.15.5.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.5.2}Description}{49}{subsection.15.5.2}} @@ -299,24 +314,24 @@ \newlabel{man/ctrl:exit-codes}{{15.5.4}{50}{Exit Codes}{subsection.15.5.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.5.5}See Also}{50}{subsection.15.5.5}} \newlabel{man/ctrl:see-also}{{15.5.5}{50}{See Also}{subsection.15.5.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.6}The \textbf {s3qlcp} command}{50}{section.15.6}} -\newlabel{man/cp:the-command-command}{{15.6}{50}{The \textbf {s3qlcp} command}{section.15.6}{}} -\newlabel{man/cp::doc}{{15.6}{50}{The \textbf {s3qlcp} command}{section.15.6}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.6}The \textbf {\texttt {s3qlcp}} command}{50}{section.15.6}} +\newlabel{man/cp:the-command-command}{{15.6}{50}{The \textbf {\texttt {s3qlcp}} command}{section.15.6}{}} +\newlabel{man/cp::doc}{{15.6}{50}{The \textbf {\texttt {s3qlcp}} command}{section.15.6}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.6.1}Synopsis}{50}{subsection.15.6.1}} \newlabel{man/cp:synopsis}{{15.6.1}{50}{Synopsis}{subsection.15.6.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.6.2}Description}{51}{subsection.15.6.2}} \newlabel{man/cp:description}{{15.6.2}{51}{Description}{subsection.15.6.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.21}} -\newlabel{man/cp:snapshotting-vs-hardlinking}{{15.6.2}{51}{Snapshotting vs Hardlinking}{subsubsection*.21}{}} +\@writefile{toc}{\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.22}} +\newlabel{man/cp:snapshotting-vs-hardlinking}{{15.6.2}{51}{Snapshotting vs Hardlinking}{subsubsection*.22}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.6.3}Options}{51}{subsection.15.6.3}} \newlabel{man/cp:options}{{15.6.3}{51}{Options}{subsection.15.6.3}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.6.4}Exit Codes}{51}{subsection.15.6.4}} \newlabel{man/cp:exit-codes}{{15.6.4}{51}{Exit Codes}{subsection.15.6.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.6.5}See Also}{52}{subsection.15.6.5}} \newlabel{man/cp:see-also}{{15.6.5}{52}{See Also}{subsection.15.6.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.7}The \textbf {s3qlrm} command}{52}{section.15.7}} -\newlabel{man/rm:the-command-command}{{15.7}{52}{The \textbf {s3qlrm} command}{section.15.7}{}} -\newlabel{man/rm::doc}{{15.7}{52}{The \textbf {s3qlrm} command}{section.15.7}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.7}The \textbf {\texttt {s3qlrm}} command}{52}{section.15.7}} +\newlabel{man/rm:the-command-command}{{15.7}{52}{The \textbf {\texttt {s3qlrm}} command}{section.15.7}{}} +\newlabel{man/rm::doc}{{15.7}{52}{The \textbf {\texttt {s3qlrm}} command}{section.15.7}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.7.1}Synopsis}{52}{subsection.15.7.1}} \newlabel{man/rm:synopsis}{{15.7.1}{52}{Synopsis}{subsection.15.7.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.7.2}Description}{52}{subsection.15.7.2}} @@ -327,9 +342,9 @@ \newlabel{man/rm:exit-codes}{{15.7.4}{52}{Exit Codes}{subsection.15.7.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.7.5}See Also}{53}{subsection.15.7.5}} \newlabel{man/rm:see-also}{{15.7.5}{53}{See Also}{subsection.15.7.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.8}The \textbf {s3qllock} command}{53}{section.15.8}} -\newlabel{man/lock:the-command-command}{{15.8}{53}{The \textbf {s3qllock} command}{section.15.8}{}} -\newlabel{man/lock::doc}{{15.8}{53}{The \textbf {s3qllock} command}{section.15.8}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.8}The \textbf {\texttt {s3qllock}} command}{53}{section.15.8}} +\newlabel{man/lock:the-command-command}{{15.8}{53}{The \textbf {\texttt {s3qllock}} command}{section.15.8}{}} +\newlabel{man/lock::doc}{{15.8}{53}{The \textbf {\texttt {s3qllock}} command}{section.15.8}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.8.1}Synopsis}{53}{subsection.15.8.1}} \newlabel{man/lock:synopsis}{{15.8.1}{53}{Synopsis}{subsection.15.8.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.8.2}Description}{53}{subsection.15.8.2}} @@ -342,9 +357,9 @@ \newlabel{man/lock:exit-codes}{{15.8.5}{54}{Exit Codes}{subsection.15.8.5}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.8.6}See Also}{54}{subsection.15.8.6}} \newlabel{man/lock:see-also}{{15.8.6}{54}{See Also}{subsection.15.8.6}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.9}The \textbf {umount.s3ql} command}{54}{section.15.9}} -\newlabel{man/umount:the-command-command}{{15.9}{54}{The \textbf {umount.s3ql} command}{section.15.9}{}} -\newlabel{man/umount::doc}{{15.9}{54}{The \textbf {umount.s3ql} command}{section.15.9}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.9}The \textbf {\texttt {umount.s3ql}} command}{54}{section.15.9}} +\newlabel{man/umount:the-command-command}{{15.9}{54}{The \textbf {\texttt {umount.s3ql}} command}{section.15.9}{}} +\newlabel{man/umount::doc}{{15.9}{54}{The \textbf {\texttt {umount.s3ql}} command}{section.15.9}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.9.1}Synopsis}{54}{subsection.15.9.1}} \newlabel{man/umount:synopsis}{{15.9.1}{54}{Synopsis}{subsection.15.9.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.9.2}Description}{54}{subsection.15.9.2}} @@ -355,9 +370,9 @@ \newlabel{man/umount:exit-codes}{{15.9.4}{55}{Exit Codes}{subsection.15.9.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.9.5}See Also}{55}{subsection.15.9.5}} \newlabel{man/umount:see-also}{{15.9.5}{55}{See Also}{subsection.15.9.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.10}The \textbf {fsck.s3ql} command}{55}{section.15.10}} -\newlabel{man/fsck:the-command-command}{{15.10}{55}{The \textbf {fsck.s3ql} command}{section.15.10}{}} -\newlabel{man/fsck::doc}{{15.10}{55}{The \textbf {fsck.s3ql} command}{section.15.10}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.10}The \textbf {\texttt {fsck.s3ql}} command}{55}{section.15.10}} +\newlabel{man/fsck:the-command-command}{{15.10}{55}{The \textbf {\texttt {fsck.s3ql}} command}{section.15.10}{}} +\newlabel{man/fsck::doc}{{15.10}{55}{The \textbf {\texttt {fsck.s3ql}} command}{section.15.10}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.10.1}Synopsis}{55}{subsection.15.10.1}} \newlabel{man/fsck:synopsis}{{15.10.1}{55}{Synopsis}{subsection.15.10.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.10.2}Description}{55}{subsection.15.10.2}} @@ -368,10 +383,10 @@ \newlabel{man/fsck:exit-codes}{{15.10.4}{56}{Exit Codes}{subsection.15.10.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.10.5}See Also}{57}{subsection.15.10.5}} \newlabel{man/fsck:see-also}{{15.10.5}{57}{See Also}{subsection.15.10.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {s3ql\_oauth\_client} command}{57}{section.15.11}} -\newlabel{man/oauth_client:the-command-command}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} -\newlabel{man/oauth_client:oauth-client}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} -\newlabel{man/oauth_client::doc}{{15.11}{57}{The \textbf {s3ql\_oauth\_client} command}{section.15.11}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{57}{section.15.11}} +\newlabel{man/oauth_client:the-command-command}{{15.11}{57}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} +\newlabel{man/oauth_client::doc}{{15.11}{57}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} +\newlabel{man/oauth_client:oauth-client}{{15.11}{57}{The \textbf {\texttt {s3ql\_oauth\_client}} command}{section.15.11}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1}} \newlabel{man/oauth_client:synopsis}{{15.11.1}{57}{Synopsis}{subsection.15.11.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2}} @@ -382,9 +397,9 @@ \newlabel{man/oauth_client:exit-codes}{{15.11.4}{57}{Exit Codes}{subsection.15.11.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.11.5}See Also}{58}{subsection.15.11.5}} \newlabel{man/oauth_client:see-also}{{15.11.5}{58}{See Also}{subsection.15.11.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.12}The \textbf {s3ql\_verify} command}{58}{section.15.12}} -\newlabel{man/verify:the-command-command}{{15.12}{58}{The \textbf {s3ql\_verify} command}{section.15.12}{}} -\newlabel{man/verify::doc}{{15.12}{58}{The \textbf {s3ql\_verify} command}{section.15.12}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.12}The \textbf {\texttt {s3ql\_verify}} command}{58}{section.15.12}} +\newlabel{man/verify:the-command-command}{{15.12}{58}{The \textbf {\texttt {s3ql\_verify}} command}{section.15.12}{}} +\newlabel{man/verify::doc}{{15.12}{58}{The \textbf {\texttt {s3ql\_verify}} command}{section.15.12}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.12.1}Synopsis}{58}{subsection.15.12.1}} \newlabel{man/verify:synopsis}{{15.12.1}{58}{Synopsis}{subsection.15.12.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.12.2}Description}{58}{subsection.15.12.2}} @@ -395,9 +410,9 @@ \newlabel{man/verify:exit-codes}{{15.12.4}{59}{Exit Codes}{subsection.15.12.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.12.5}See Also}{59}{subsection.15.12.5}} \newlabel{man/verify:see-also}{{15.12.5}{59}{See Also}{subsection.15.12.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.13}The \textbf {pcp} command}{59}{section.15.13}} -\newlabel{man/pcp:the-command-command}{{15.13}{59}{The \textbf {pcp} command}{section.15.13}{}} -\newlabel{man/pcp::doc}{{15.13}{59}{The \textbf {pcp} command}{section.15.13}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.13}The \textbf {\texttt {pcp}} command}{59}{section.15.13}} +\newlabel{man/pcp:the-command-command}{{15.13}{59}{The \textbf {\texttt {pcp}} command}{section.15.13}{}} +\newlabel{man/pcp::doc}{{15.13}{59}{The \textbf {\texttt {pcp}} command}{section.15.13}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.13.1}Synopsis}{59}{subsection.15.13.1}} \newlabel{man/pcp:synopsis}{{15.13.1}{59}{Synopsis}{subsection.15.13.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.13.2}Description}{59}{subsection.15.13.2}} @@ -408,9 +423,9 @@ \newlabel{man/pcp:exit-codes}{{15.13.4}{60}{Exit Codes}{subsection.15.13.4}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.13.5}See Also}{60}{subsection.15.13.5}} \newlabel{man/pcp:see-also}{{15.13.5}{60}{See Also}{subsection.15.13.5}{}} -\@writefile{toc}{\contentsline {section}{\numberline {15.14}The \textbf {expire\_backups} command}{60}{section.15.14}} -\newlabel{man/expire_backups:the-command-command}{{15.14}{60}{The \textbf {expire\_backups} command}{section.15.14}{}} -\newlabel{man/expire_backups::doc}{{15.14}{60}{The \textbf {expire\_backups} command}{section.15.14}{}} +\@writefile{toc}{\contentsline {section}{\numberline {15.14}The \textbf {\texttt {expire\_backups}} command}{60}{section.15.14}} +\newlabel{man/expire_backups:the-command-command}{{15.14}{60}{The \textbf {\texttt {expire\_backups}} command}{section.15.14}{}} +\newlabel{man/expire_backups::doc}{{15.14}{60}{The \textbf {\texttt {expire\_backups}} command}{section.15.14}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.14.1}Synopsis}{60}{subsection.15.14.1}} \newlabel{man/expire_backups:synopsis}{{15.14.1}{60}{Synopsis}{subsection.15.14.1}{}} \@writefile{toc}{\contentsline {subsection}{\numberline {15.14.2}Description}{60}{subsection.15.14.2}} @@ -424,21 +439,23 @@ \@writefile{toc}{\contentsline {chapter}{\numberline {16}Further Resources / Getting Help}{63}{chapter.16}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{resources:resources}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} -\newlabel{resources:further-resources-getting-help}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} \newlabel{resources::doc}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} +\newlabel{resources:further-resources-getting-help}{{16}{63}{Further Resources / Getting Help}{chapter.16}{}} \@writefile{toc}{\contentsline {chapter}{\numberline {17}Implementation Details}{65}{chapter.17}} \@writefile{lof}{\addvspace {10\p@ }} \@writefile{lot}{\addvspace {10\p@ }} +\@writefile{loliteral-block}{\addvspace {10\p@ }} \newlabel{impl_details:impl-details}{{17}{65}{Implementation Details}{chapter.17}{}} -\newlabel{impl_details:implementation-details}{{17}{65}{Implementation Details}{chapter.17}{}} \newlabel{impl_details::doc}{{17}{65}{Implementation Details}{chapter.17}{}} +\newlabel{impl_details:implementation-details}{{17}{65}{Implementation Details}{chapter.17}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1}} \newlabel{impl_details:metadata-storage}{{17.1}{65}{Metadata Storage}{section.17.1}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2}} \newlabel{impl_details:data-storage}{{17.2}{65}{Data Storage}{section.17.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {17.3}Data De-Duplication}{65}{section.17.3}} -\newlabel{impl_details:data-de-duplication}{{17.3}{65}{Data De-Duplication}{section.17.3}{}} +\@writefile{toc}{\contentsline {section}{\numberline {17.3}Data De-Duplication}{66}{section.17.3}} +\newlabel{impl_details:data-de-duplication}{{17.3}{66}{Data De-Duplication}{section.17.3}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.4}Caching}{66}{section.17.4}} \newlabel{impl_details:caching}{{17.4}{66}{Caching}{section.17.4}{}} \@writefile{toc}{\contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{66}{section.17.5}} diff -Nru s3ql-2.22+dfsg/doc/latex/manual.idx s3ql-2.23+dfsg/doc/latex/manual.idx --- s3ql-2.22+dfsg/doc/latex/manual.idx 2017-06-23 16:36:56.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/manual.idx 2017-08-18 18:11:57.000000000 +0000 @@ -24,6 +24,8 @@ \indexentry{tcp-timeout!swift\_backend command line option|hyperpage}{9} \indexentry{swift\_backend command line option!disable-expect100|hyperpage}{9} \indexentry{disable-expect100!swift\_backend command line option|hyperpage}{9} +\indexentry{swift\_backend command line option!no-feature-detection|hyperpage}{9} +\indexentry{no-feature-detection!swift\_backend command line option|hyperpage}{9} \indexentry{s3c\_backend command line option!no-ssl|hyperpage}{10} \indexentry{no-ssl!s3c\_backend command line option|hyperpage}{10} \indexentry{s3c\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}|hyperpage}{10} diff -Nru s3ql-2.22+dfsg/doc/latex/manual.out s3ql-2.23+dfsg/doc/latex/manual.out --- s3ql-2.22+dfsg/doc/latex/manual.out 2017-06-23 16:36:56.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/manual.out 2017-08-18 18:11:57.000000000 +0000 @@ -1,80 +1,80 @@ -\BOOKMARK [0][-]{chapter.1}{About S3QL}{}% 1 -\BOOKMARK [1][-]{section.1.1}{Features}{chapter.1}% 2 -\BOOKMARK [1][-]{section.1.2}{Development Status}{chapter.1}% 3 -\BOOKMARK [1][-]{section.1.3}{Supported Platforms}{chapter.1}% 4 -\BOOKMARK [1][-]{section.1.4}{Contributing}{chapter.1}% 5 -\BOOKMARK [0][-]{chapter.2}{Installation}{}% 6 -\BOOKMARK [1][-]{section.2.1}{Dependencies}{chapter.2}% 7 -\BOOKMARK [1][-]{section.2.2}{Installing S3QL}{chapter.2}% 8 -\BOOKMARK [1][-]{section.2.3}{Development Version}{chapter.2}% 9 -\BOOKMARK [1][-]{section.2.4}{Running tests requiring remote servers}{chapter.2}% 10 -\BOOKMARK [0][-]{chapter.3}{Storage Backends}{}% 11 -\BOOKMARK [1][-]{section.3.1}{Google Storage}{chapter.3}% 12 -\BOOKMARK [1][-]{section.3.2}{Amazon S3}{chapter.3}% 13 -\BOOKMARK [1][-]{section.3.3}{OpenStack/Swift}{chapter.3}% 14 -\BOOKMARK [1][-]{section.3.4}{Rackspace CloudFiles}{chapter.3}% 15 -\BOOKMARK [1][-]{section.3.5}{S3 compatible}{chapter.3}% 16 -\BOOKMARK [1][-]{section.3.6}{Local}{chapter.3}% 17 -\BOOKMARK [0][-]{chapter.4}{Important Rules to Avoid Losing Data}{}% 18 -\BOOKMARK [1][-]{section.4.1}{Rules in a Nutshell}{chapter.4}% 19 -\BOOKMARK [1][-]{section.4.2}{Consistency Window List}{chapter.4}% 20 -\BOOKMARK [1][-]{section.4.3}{Data Consistency}{chapter.4}% 21 -\BOOKMARK [1][-]{section.4.4}{Data Durability}{chapter.4}% 22 -\BOOKMARK [0][-]{chapter.5}{File System Creation}{}% 23 -\BOOKMARK [0][-]{chapter.6}{Managing File Systems}{}% 24 -\BOOKMARK [1][-]{section.6.1}{Changing the Passphrase}{chapter.6}% 25 -\BOOKMARK [1][-]{section.6.2}{Upgrading the file system}{chapter.6}% 26 -\BOOKMARK [1][-]{section.6.3}{Deleting a file system}{chapter.6}% 27 -\BOOKMARK [1][-]{section.6.4}{Restoring Metadata Backups}{chapter.6}% 28 -\BOOKMARK [0][-]{chapter.7}{Mounting}{}% 29 -\BOOKMARK [1][-]{section.7.1}{Permission Checking}{chapter.7}% 30 -\BOOKMARK [1][-]{section.7.2}{Compression Algorithms}{chapter.7}% 31 -\BOOKMARK [1][-]{section.7.3}{Notes about Caching}{chapter.7}% 32 -\BOOKMARK [1][-]{section.7.4}{Failure Modes}{chapter.7}% 33 -\BOOKMARK [1][-]{section.7.5}{Automatic Mounting}{chapter.7}% 34 -\BOOKMARK [0][-]{chapter.8}{Advanced S3QL Features}{}% 35 -\BOOKMARK [1][-]{section.8.1}{Snapshotting and Copy-on-Write}{chapter.8}% 36 -\BOOKMARK [1][-]{section.8.2}{Getting Statistics}{chapter.8}% 37 -\BOOKMARK [1][-]{section.8.3}{Immutable Trees}{chapter.8}% 38 -\BOOKMARK [1][-]{section.8.4}{Fast Recursive Removal}{chapter.8}% 39 -\BOOKMARK [1][-]{section.8.5}{Runtime Configuration}{chapter.8}% 40 -\BOOKMARK [0][-]{chapter.9}{Unmounting}{}% 41 -\BOOKMARK [0][-]{chapter.10}{Checking for Errors}{}% 42 -\BOOKMARK [1][-]{section.10.1}{Checking and repairing internal file system errors}{chapter.10}% 43 -\BOOKMARK [1][-]{section.10.2}{Detecting and handling backend data corruption}{chapter.10}% 44 -\BOOKMARK [0][-]{chapter.11}{Storing Authentication Information}{}% 45 -\BOOKMARK [0][-]{chapter.12}{Contributed Programs}{}% 46 -\BOOKMARK [1][-]{section.12.1}{benchmark.py}{chapter.12}% 47 -\BOOKMARK [1][-]{section.12.2}{clone\137fs.py}{chapter.12}% 48 -\BOOKMARK [1][-]{section.12.3}{pcp.py}{chapter.12}% 49 -\BOOKMARK [1][-]{section.12.4}{s3ql\137backup.sh}{chapter.12}% 50 -\BOOKMARK [1][-]{section.12.5}{expire\137backups.py}{chapter.12}% 51 -\BOOKMARK [1][-]{section.12.6}{remove\137objects.py}{chapter.12}% 52 -\BOOKMARK [0][-]{chapter.13}{Tips \046 Tricks}{}% 53 -\BOOKMARK [1][-]{section.13.1}{SSH Backend}{chapter.13}% 54 -\BOOKMARK [1][-]{section.13.2}{Permanently mounted backup file system}{chapter.13}% 55 -\BOOKMARK [1][-]{section.13.3}{Improving copy performance}{chapter.13}% 56 -\BOOKMARK [0][-]{chapter.14}{Known Issues}{}% 57 -\BOOKMARK [0][-]{chapter.15}{Manpages}{}% 58 -\BOOKMARK [1][-]{section.15.1}{The mkfs.s3ql command}{chapter.15}% 59 -\BOOKMARK [1][-]{section.15.2}{The s3qladm command}{chapter.15}% 60 -\BOOKMARK [1][-]{section.15.3}{The mount.s3ql command}{chapter.15}% 61 -\BOOKMARK [1][-]{section.15.4}{The s3qlstat command}{chapter.15}% 62 -\BOOKMARK [1][-]{section.15.5}{The s3qlctrl command}{chapter.15}% 63 -\BOOKMARK [1][-]{section.15.6}{The s3qlcp command}{chapter.15}% 64 -\BOOKMARK [1][-]{section.15.7}{The s3qlrm command}{chapter.15}% 65 -\BOOKMARK [1][-]{section.15.8}{The s3qllock command}{chapter.15}% 66 -\BOOKMARK [1][-]{section.15.9}{The umount.s3ql command}{chapter.15}% 67 -\BOOKMARK [1][-]{section.15.10}{The fsck.s3ql command}{chapter.15}% 68 -\BOOKMARK [1][-]{section.15.11}{The s3ql\137oauth\137client command}{chapter.15}% 69 -\BOOKMARK [1][-]{section.15.12}{The s3ql\137verify command}{chapter.15}% 70 -\BOOKMARK [1][-]{section.15.13}{The pcp command}{chapter.15}% 71 -\BOOKMARK [1][-]{section.15.14}{The expire\137backups command}{chapter.15}% 72 -\BOOKMARK [0][-]{chapter.16}{Further Resources / Getting Help}{}% 73 -\BOOKMARK [0][-]{chapter.17}{Implementation Details}{}% 74 -\BOOKMARK [1][-]{section.17.1}{Metadata Storage}{chapter.17}% 75 -\BOOKMARK [1][-]{section.17.2}{Data Storage}{chapter.17}% 76 -\BOOKMARK [1][-]{section.17.3}{Data De-Duplication}{chapter.17}% 77 -\BOOKMARK [1][-]{section.17.4}{Caching}{chapter.17}% 78 -\BOOKMARK [1][-]{section.17.5}{Eventual Consistency Handling}{chapter.17}% 79 -\BOOKMARK [1][-]{section.17.6}{Encryption}{chapter.17}% 80 +\BOOKMARK [0][-]{chapter.1}{\376\377\000A\000b\000o\000u\000t\000\040\000S\0003\000Q\000L}{}% 1 +\BOOKMARK [1][-]{section.1.1}{\376\377\000F\000e\000a\000t\000u\000r\000e\000s}{chapter.1}% 2 +\BOOKMARK [1][-]{section.1.2}{\376\377\000D\000e\000v\000e\000l\000o\000p\000m\000e\000n\000t\000\040\000S\000t\000a\000t\000u\000s}{chapter.1}% 3 +\BOOKMARK [1][-]{section.1.3}{\376\377\000S\000u\000p\000p\000o\000r\000t\000e\000d\000\040\000P\000l\000a\000t\000f\000o\000r\000m\000s}{chapter.1}% 4 +\BOOKMARK [1][-]{section.1.4}{\376\377\000C\000o\000n\000t\000r\000i\000b\000u\000t\000i\000n\000g}{chapter.1}% 5 +\BOOKMARK [0][-]{chapter.2}{\376\377\000I\000n\000s\000t\000a\000l\000l\000a\000t\000i\000o\000n}{}% 6 +\BOOKMARK [1][-]{section.2.1}{\376\377\000D\000e\000p\000e\000n\000d\000e\000n\000c\000i\000e\000s}{chapter.2}% 7 +\BOOKMARK [1][-]{section.2.2}{\376\377\000I\000n\000s\000t\000a\000l\000l\000i\000n\000g\000\040\000S\0003\000Q\000L}{chapter.2}% 8 +\BOOKMARK [1][-]{section.2.3}{\376\377\000D\000e\000v\000e\000l\000o\000p\000m\000e\000n\000t\000\040\000V\000e\000r\000s\000i\000o\000n}{chapter.2}% 9 +\BOOKMARK [1][-]{section.2.4}{\376\377\000R\000u\000n\000n\000i\000n\000g\000\040\000t\000e\000s\000t\000s\000\040\000r\000e\000q\000u\000i\000r\000i\000n\000g\000\040\000r\000e\000m\000o\000t\000e\000\040\000s\000e\000r\000v\000e\000r\000s}{chapter.2}% 10 +\BOOKMARK [0][-]{chapter.3}{\376\377\000S\000t\000o\000r\000a\000g\000e\000\040\000B\000a\000c\000k\000e\000n\000d\000s}{}% 11 +\BOOKMARK [1][-]{section.3.1}{\376\377\000G\000o\000o\000g\000l\000e\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.3}% 12 +\BOOKMARK [1][-]{section.3.2}{\376\377\000A\000m\000a\000z\000o\000n\000\040\000S\0003}{chapter.3}% 13 +\BOOKMARK [1][-]{section.3.3}{\376\377\000O\000p\000e\000n\000S\000t\000a\000c\000k\000/\000S\000w\000i\000f\000t}{chapter.3}% 14 +\BOOKMARK [1][-]{section.3.4}{\376\377\000R\000a\000c\000k\000s\000p\000a\000c\000e\000\040\000C\000l\000o\000u\000d\000F\000i\000l\000e\000s}{chapter.3}% 15 +\BOOKMARK [1][-]{section.3.5}{\376\377\000S\0003\000\040\000c\000o\000m\000p\000a\000t\000i\000b\000l\000e}{chapter.3}% 16 +\BOOKMARK [1][-]{section.3.6}{\376\377\000L\000o\000c\000a\000l}{chapter.3}% 17 +\BOOKMARK [0][-]{chapter.4}{\376\377\000I\000m\000p\000o\000r\000t\000a\000n\000t\000\040\000R\000u\000l\000e\000s\000\040\000t\000o\000\040\000A\000v\000o\000i\000d\000\040\000L\000o\000s\000i\000n\000g\000\040\000D\000a\000t\000a}{}% 18 +\BOOKMARK [1][-]{section.4.1}{\376\377\000R\000u\000l\000e\000s\000\040\000i\000n\000\040\000a\000\040\000N\000u\000t\000s\000h\000e\000l\000l}{chapter.4}% 19 +\BOOKMARK [1][-]{section.4.2}{\376\377\000C\000o\000n\000s\000i\000s\000t\000e\000n\000c\000y\000\040\000W\000i\000n\000d\000o\000w\000\040\000L\000i\000s\000t}{chapter.4}% 20 +\BOOKMARK [1][-]{section.4.3}{\376\377\000D\000a\000t\000a\000\040\000C\000o\000n\000s\000i\000s\000t\000e\000n\000c\000y}{chapter.4}% 21 +\BOOKMARK [1][-]{section.4.4}{\376\377\000D\000a\000t\000a\000\040\000D\000u\000r\000a\000b\000i\000l\000i\000t\000y}{chapter.4}% 22 +\BOOKMARK [0][-]{chapter.5}{\376\377\000F\000i\000l\000e\000\040\000S\000y\000s\000t\000e\000m\000\040\000C\000r\000e\000a\000t\000i\000o\000n}{}% 23 +\BOOKMARK [0][-]{chapter.6}{\376\377\000M\000a\000n\000a\000g\000i\000n\000g\000\040\000F\000i\000l\000e\000\040\000S\000y\000s\000t\000e\000m\000s}{}% 24 +\BOOKMARK [1][-]{section.6.1}{\376\377\000C\000h\000a\000n\000g\000i\000n\000g\000\040\000t\000h\000e\000\040\000P\000a\000s\000s\000p\000h\000r\000a\000s\000e}{chapter.6}% 25 +\BOOKMARK [1][-]{section.6.2}{\376\377\000U\000p\000g\000r\000a\000d\000i\000n\000g\000\040\000t\000h\000e\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m}{chapter.6}% 26 +\BOOKMARK [1][-]{section.6.3}{\376\377\000D\000e\000l\000e\000t\000i\000n\000g\000\040\000a\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m}{chapter.6}% 27 +\BOOKMARK [1][-]{section.6.4}{\376\377\000R\000e\000s\000t\000o\000r\000i\000n\000g\000\040\000M\000e\000t\000a\000d\000a\000t\000a\000\040\000B\000a\000c\000k\000u\000p\000s}{chapter.6}% 28 +\BOOKMARK [0][-]{chapter.7}{\376\377\000M\000o\000u\000n\000t\000i\000n\000g}{}% 29 +\BOOKMARK [1][-]{section.7.1}{\376\377\000P\000e\000r\000m\000i\000s\000s\000i\000o\000n\000\040\000C\000h\000e\000c\000k\000i\000n\000g}{chapter.7}% 30 +\BOOKMARK [1][-]{section.7.2}{\376\377\000C\000o\000m\000p\000r\000e\000s\000s\000i\000o\000n\000\040\000A\000l\000g\000o\000r\000i\000t\000h\000m\000s}{chapter.7}% 31 +\BOOKMARK [1][-]{section.7.3}{\376\377\000N\000o\000t\000e\000s\000\040\000a\000b\000o\000u\000t\000\040\000C\000a\000c\000h\000i\000n\000g}{chapter.7}% 32 +\BOOKMARK [1][-]{section.7.4}{\376\377\000F\000a\000i\000l\000u\000r\000e\000\040\000M\000o\000d\000e\000s}{chapter.7}% 33 +\BOOKMARK [1][-]{section.7.5}{\376\377\000A\000u\000t\000o\000m\000a\000t\000i\000c\000\040\000M\000o\000u\000n\000t\000i\000n\000g}{chapter.7}% 34 +\BOOKMARK [0][-]{chapter.8}{\376\377\000A\000d\000v\000a\000n\000c\000e\000d\000\040\000S\0003\000Q\000L\000\040\000F\000e\000a\000t\000u\000r\000e\000s}{}% 35 +\BOOKMARK [1][-]{section.8.1}{\376\377\000S\000n\000a\000p\000s\000h\000o\000t\000t\000i\000n\000g\000\040\000a\000n\000d\000\040\000C\000o\000p\000y\000-\000o\000n\000-\000W\000r\000i\000t\000e}{chapter.8}% 36 +\BOOKMARK [1][-]{section.8.2}{\376\377\000G\000e\000t\000t\000i\000n\000g\000\040\000S\000t\000a\000t\000i\000s\000t\000i\000c\000s}{chapter.8}% 37 +\BOOKMARK [1][-]{section.8.3}{\376\377\000I\000m\000m\000u\000t\000a\000b\000l\000e\000\040\000T\000r\000e\000e\000s}{chapter.8}% 38 +\BOOKMARK [1][-]{section.8.4}{\376\377\000F\000a\000s\000t\000\040\000R\000e\000c\000u\000r\000s\000i\000v\000e\000\040\000R\000e\000m\000o\000v\000a\000l}{chapter.8}% 39 +\BOOKMARK [1][-]{section.8.5}{\376\377\000R\000u\000n\000t\000i\000m\000e\000\040\000C\000o\000n\000f\000i\000g\000u\000r\000a\000t\000i\000o\000n}{chapter.8}% 40 +\BOOKMARK [0][-]{chapter.9}{\376\377\000U\000n\000m\000o\000u\000n\000t\000i\000n\000g}{}% 41 +\BOOKMARK [0][-]{chapter.10}{\376\377\000C\000h\000e\000c\000k\000i\000n\000g\000\040\000f\000o\000r\000\040\000E\000r\000r\000o\000r\000s}{}% 42 +\BOOKMARK [1][-]{section.10.1}{\376\377\000C\000h\000e\000c\000k\000i\000n\000g\000\040\000a\000n\000d\000\040\000r\000e\000p\000a\000i\000r\000i\000n\000g\000\040\000i\000n\000t\000e\000r\000n\000a\000l\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m\000\040\000e\000r\000r\000o\000r\000s}{chapter.10}% 43 +\BOOKMARK [1][-]{section.10.2}{\376\377\000D\000e\000t\000e\000c\000t\000i\000n\000g\000\040\000a\000n\000d\000\040\000h\000a\000n\000d\000l\000i\000n\000g\000\040\000b\000a\000c\000k\000e\000n\000d\000\040\000d\000a\000t\000a\000\040\000c\000o\000r\000r\000u\000p\000t\000i\000o\000n}{chapter.10}% 44 +\BOOKMARK [0][-]{chapter.11}{\376\377\000S\000t\000o\000r\000i\000n\000g\000\040\000A\000u\000t\000h\000e\000n\000t\000i\000c\000a\000t\000i\000o\000n\000\040\000I\000n\000f\000o\000r\000m\000a\000t\000i\000o\000n}{}% 45 +\BOOKMARK [0][-]{chapter.12}{\376\377\000C\000o\000n\000t\000r\000i\000b\000u\000t\000e\000d\000\040\000P\000r\000o\000g\000r\000a\000m\000s}{}% 46 +\BOOKMARK [1][-]{section.12.1}{\376\377\000b\000e\000n\000c\000h\000m\000a\000r\000k\000.\000p\000y}{chapter.12}% 47 +\BOOKMARK [1][-]{section.12.2}{\376\377\000c\000l\000o\000n\000e\000\137\000f\000s\000.\000p\000y}{chapter.12}% 48 +\BOOKMARK [1][-]{section.12.3}{\376\377\000p\000c\000p\000.\000p\000y}{chapter.12}% 49 +\BOOKMARK [1][-]{section.12.4}{\376\377\000s\0003\000q\000l\000\137\000b\000a\000c\000k\000u\000p\000.\000s\000h}{chapter.12}% 50 +\BOOKMARK [1][-]{section.12.5}{\376\377\000e\000x\000p\000i\000r\000e\000\137\000b\000a\000c\000k\000u\000p\000s\000.\000p\000y}{chapter.12}% 51 +\BOOKMARK [1][-]{section.12.6}{\376\377\000r\000e\000m\000o\000v\000e\000\137\000o\000b\000j\000e\000c\000t\000s\000.\000p\000y}{chapter.12}% 52 +\BOOKMARK [0][-]{chapter.13}{\376\377\000T\000i\000p\000s\000\040\000\046\000\040\000T\000r\000i\000c\000k\000s}{}% 53 +\BOOKMARK [1][-]{section.13.1}{\376\377\000S\000S\000H\000\040\000B\000a\000c\000k\000e\000n\000d}{chapter.13}% 54 +\BOOKMARK [1][-]{section.13.2}{\376\377\000P\000e\000r\000m\000a\000n\000e\000n\000t\000l\000y\000\040\000m\000o\000u\000n\000t\000e\000d\000\040\000b\000a\000c\000k\000u\000p\000\040\000f\000i\000l\000e\000\040\000s\000y\000s\000t\000e\000m}{chapter.13}% 55 +\BOOKMARK [1][-]{section.13.3}{\376\377\000I\000m\000p\000r\000o\000v\000i\000n\000g\000\040\000c\000o\000p\000y\000\040\000p\000e\000r\000f\000o\000r\000m\000a\000n\000c\000e}{chapter.13}% 56 +\BOOKMARK [0][-]{chapter.14}{\376\377\000K\000n\000o\000w\000n\000\040\000I\000s\000s\000u\000e\000s}{}% 57 +\BOOKMARK [0][-]{chapter.15}{\376\377\000M\000a\000n\000p\000a\000g\000e\000s}{}% 58 +\BOOKMARK [1][-]{section.15.1}{\376\377\000T\000h\000e\000\040\000m\000k\000f\000s\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 59 +\BOOKMARK [1][-]{section.15.2}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000a\000d\000m\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 60 +\BOOKMARK [1][-]{section.15.3}{\376\377\000T\000h\000e\000\040\000m\000o\000u\000n\000t\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 61 +\BOOKMARK [1][-]{section.15.4}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000s\000t\000a\000t\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 62 +\BOOKMARK [1][-]{section.15.5}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000c\000t\000r\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 63 +\BOOKMARK [1][-]{section.15.6}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000c\000p\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 64 +\BOOKMARK [1][-]{section.15.7}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000r\000m\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 65 +\BOOKMARK [1][-]{section.15.8}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000l\000o\000c\000k\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 66 +\BOOKMARK [1][-]{section.15.9}{\376\377\000T\000h\000e\000\040\000u\000m\000o\000u\000n\000t\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 67 +\BOOKMARK [1][-]{section.15.10}{\376\377\000T\000h\000e\000\040\000f\000s\000c\000k\000.\000s\0003\000q\000l\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 68 +\BOOKMARK [1][-]{section.15.11}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000\137\000o\000a\000u\000t\000h\000\137\000c\000l\000i\000e\000n\000t\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 69 +\BOOKMARK [1][-]{section.15.12}{\376\377\000T\000h\000e\000\040\000s\0003\000q\000l\000\137\000v\000e\000r\000i\000f\000y\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 70 +\BOOKMARK [1][-]{section.15.13}{\376\377\000T\000h\000e\000\040\000p\000c\000p\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 71 +\BOOKMARK [1][-]{section.15.14}{\376\377\000T\000h\000e\000\040\000e\000x\000p\000i\000r\000e\000\137\000b\000a\000c\000k\000u\000p\000s\000\040\000c\000o\000m\000m\000a\000n\000d}{chapter.15}% 72 +\BOOKMARK [0][-]{chapter.16}{\376\377\000F\000u\000r\000t\000h\000e\000r\000\040\000R\000e\000s\000o\000u\000r\000c\000e\000s\000\040\000/\000\040\000G\000e\000t\000t\000i\000n\000g\000\040\000H\000e\000l\000p}{}% 73 +\BOOKMARK [0][-]{chapter.17}{\376\377\000I\000m\000p\000l\000e\000m\000e\000n\000t\000a\000t\000i\000o\000n\000\040\000D\000e\000t\000a\000i\000l\000s}{}% 74 +\BOOKMARK [1][-]{section.17.1}{\376\377\000M\000e\000t\000a\000d\000a\000t\000a\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.17}% 75 +\BOOKMARK [1][-]{section.17.2}{\376\377\000D\000a\000t\000a\000\040\000S\000t\000o\000r\000a\000g\000e}{chapter.17}% 76 +\BOOKMARK [1][-]{section.17.3}{\376\377\000D\000a\000t\000a\000\040\000D\000e\000-\000D\000u\000p\000l\000i\000c\000a\000t\000i\000o\000n}{chapter.17}% 77 +\BOOKMARK [1][-]{section.17.4}{\376\377\000C\000a\000c\000h\000i\000n\000g}{chapter.17}% 78 +\BOOKMARK [1][-]{section.17.5}{\376\377\000E\000v\000e\000n\000t\000u\000a\000l\000\040\000C\000o\000n\000s\000i\000s\000t\000e\000n\000c\000y\000\040\000H\000a\000n\000d\000l\000i\000n\000g}{chapter.17}% 79 +\BOOKMARK [1][-]{section.17.6}{\376\377\000E\000n\000c\000r\000y\000p\000t\000i\000o\000n}{chapter.17}% 80 diff -Nru s3ql-2.22+dfsg/doc/latex/manual.tex s3ql-2.23+dfsg/doc/latex/manual.tex --- s3ql-2.22+dfsg/doc/latex/manual.tex 2017-06-23 16:36:50.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/manual.tex 2017-08-18 18:11:49.000000000 +0000 @@ -1,21 +1,39 @@ % Generated by Sphinx. \def\sphinxdocclass{report} +\newif\ifsphinxKeepOldNames \sphinxKeepOldNamestrue \documentclass[letterpaper,10pt,english]{sphinxmanual} -\usepackage[utf8]{inputenc} -\DeclareUnicodeCharacter{00A0}{\nobreakspace} +\usepackage{iftex} + +\ifPDFTeX + \usepackage[utf8]{inputenc} +\fi +\ifdefined\DeclareUnicodeCharacter + \DeclareUnicodeCharacter{00A0}{\nobreakspace} +\fi \usepackage{cmap} \usepackage[T1]{fontenc} +\usepackage{amsmath,amssymb,amstext} \usepackage{babel} \usepackage{times} \usepackage[Bjarne]{fncychap} \usepackage{longtable} \usepackage{sphinx} \usepackage{multirow} +\usepackage{eqparbox} + + +\addto\captionsenglish{\renewcommand{\figurename}{Fig.\@ }} +\addto\captionsenglish{\renewcommand{\tablename}{Table }} +\SetupFloatingEnvironment{literal-block}{name=Listing } + +\addto\extrasenglish{\def\pageautorefname{page}} + +\setcounter{tocdepth}{1} \title{S3QL Documentation} -\date{June 23, 2017} -\release{2.22} +\date{Aug 18, 2017} +\release{2.23} \author{Nikolaus Rath} \newcommand{\sphinxlogo}{} \renewcommand{\releasename}{Release} @@ -32,67 +50,73 @@ \PYG@it{\PYG@bf{\PYG@ff{#1}}}}}}} \def\PYG#1#2{\PYG@reset\PYG@toks#1+\relax+\PYG@do{#2}} -\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} +\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} +\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} \expandafter\def\csname PYG@tok@gu\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.50,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@sa\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@no\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.40}{##1}}} -\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} -\expandafter\def\csname PYG@tok@bp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@nf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} -\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} -\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} -\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@cpf\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} +\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@sc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} -\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} -\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} -\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} \expandafter\def\csname PYG@tok@c\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} -\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} +\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} +\expandafter\def\csname PYG@tok@s1\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@nd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.33,0.33,0.33}{##1}}} +\expandafter\def\csname PYG@tok@s2\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@ch\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} \expandafter\def\csname PYG@tok@vi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.73}{##1}}} -\expandafter\def\csname PYG@tok@mf\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@kt\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.60}{##1}}} -\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} -\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} -\expandafter\def\csname PYG@tok@mo\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.27,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} -\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} -\expandafter\def\csname PYG@tok@mh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.33,0.53}{##1}}} -\expandafter\def\csname PYG@tok@cp\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.33,0.47,0.60}{##1}}} +\expandafter\def\csname PYG@tok@na\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.80}{##1}}} +\expandafter\def\csname PYG@tok@gd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.63,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@cs\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.80,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ni\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.53,0.00,0.00}{##1}}} \expandafter\def\csname PYG@tok@il\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} -\expandafter\def\csname PYG@tok@go\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@vc\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.40,0.60}{##1}}} -\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} -\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} -\expandafter\def\csname PYG@tok@m\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@se\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.40,0.40}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@gr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} -\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} -\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@gi\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.63,0.00}{##1}}} \expandafter\def\csname PYG@tok@sd\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.27,0.13}{##1}}} -\expandafter\def\csname PYG@tok@mb\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.40,0.00,0.93}{##1}}} -\expandafter\def\csname PYG@tok@o\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.20,0.20,0.20}{##1}}} +\expandafter\def\csname PYG@tok@ge\endcsname{\let\PYG@it=\textit} +\expandafter\def\csname PYG@tok@w\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.73,0.73,0.73}{##1}}} +\expandafter\def\csname PYG@tok@gs\endcsname{\let\PYG@bf=\textbf} +\expandafter\def\csname PYG@tok@kd\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@err\endcsname{\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.67,0.67}{\strut ##1}}} +\expandafter\def\csname PYG@tok@vm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@nb\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.44,0.13}{##1}}} +\expandafter\def\csname PYG@tok@sh\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@sb\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@gp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.78,0.36,0.04}{##1}}} +\expandafter\def\csname PYG@tok@nc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.73,0.00,0.40}{##1}}} +\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@vg\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.87,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@dl\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@fm\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.40,0.73}{##1}}} \expandafter\def\csname PYG@tok@s\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} +\expandafter\def\csname PYG@tok@nt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kr\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@mi\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.87}{##1}}} +\expandafter\def\csname PYG@tok@si\endcsname{\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{0.93,0.93,0.93}{\strut ##1}}} \expandafter\def\csname PYG@tok@nv\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.60,0.40,0.20}{##1}}} +\expandafter\def\csname PYG@tok@ss\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.67,0.40,0.00}{##1}}} +\expandafter\def\csname PYG@tok@k\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@ne\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{1.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@c1\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@sr\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,1.00}{\strut ##1}}} +\expandafter\def\csname PYG@tok@cm\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.53,0.53,0.53}{##1}}} +\expandafter\def\csname PYG@tok@ow\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.00}{##1}}} +\expandafter\def\csname PYG@tok@nl\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.60,0.47,0.00}{##1}}} +\expandafter\def\csname PYG@tok@gh\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.00,0.50}{##1}}} +\expandafter\def\csname PYG@tok@sx\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.87,0.13,0.00}{##1}}\def\PYG@bc##1{\setlength{\fboxsep}{0pt}\colorbox[rgb]{1.00,0.94,0.94}{\strut ##1}}} \expandafter\def\csname PYG@tok@nn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.05,0.52,0.71}{##1}}} -\expandafter\def\csname PYG@tok@kc\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} -\expandafter\def\csname PYG@tok@gt\endcsname{\def\PYG@tc##1{\textcolor[rgb]{0.00,0.27,0.87}{##1}}} +\expandafter\def\csname PYG@tok@kn\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.53,0.00}{##1}}} +\expandafter\def\csname PYG@tok@kp\endcsname{\let\PYG@bf=\textbf\def\PYG@tc##1{\textcolor[rgb]{0.00,0.20,0.53}{##1}}} \def\PYGZbs{\char`\\} \def\PYGZus{\char`\_} @@ -126,7 +150,7 @@ \chapter{About S3QL} -\label{about:s3ql-user-s-guide}\label{about:about-s3ql}\label{about::doc} +\label{about:about-s3ql}\label{about:s3ql-user-s-guide}\label{about::doc} S3QL is a file system that stores all its data online using storage services like \href{http://code.google.com/apis/storage/}{Google Storage}, \href{http://aws.amazon.com/s3}{Amazon S3}, or \href{http://openstack.org/projects/storage/}{OpenStack}. S3QL effectively provides a hard disk of dynamic, infinite capacity that @@ -232,12 +256,12 @@ \chapter{Installation} -\label{installation:github}\label{installation:installation}\label{installation::doc} +\label{installation:github}\label{installation::doc}\label{installation:installation} S3QL depends on several other programs and libraries that have to be installed first. The best method to satisfy these dependencies depends on your distribution. -The following instructions are for S3QL 2.22 and should be +The following instructions are for S3QL 2.23 and should be applicable to any system. The \href{https://bitbucket.org/nikratio/s3ql/wiki/Home}{S3QL Wiki} contains \href{https://bitbucket.org/nikratio/s3ql/wiki/Installation}{additional help} help for specific distributions and operating systems. Note, however, that @@ -313,10 +337,10 @@ \end{itemize} -To check if a specific module \code{\textless{}module\textgreater{}} is installed, execute -\code{python3 -c 'import \emph{\textless{}module\textgreater{}}; +To check if a specific module \sphinxcode{\textless{}module\textgreater{}} is installed, execute +\sphinxcode{python3 -c 'import \emph{\textless{}module\textgreater{}}; print(\emph{\textless{}module\textgreater{}}.\_\_version\_\_)'}. This will result in an -\code{ImportError} if the module is not installed, and will print the +\sphinxcode{ImportError} if the module is not installed, and will print the installed version if the module is installed. \end{itemize} @@ -327,16 +351,16 @@ To build and install S3QL itself, proceed as follows: \begin{enumerate} \item {} -Download S3QL from \href{https://bitbucket.org/nikratio/s3ql/downloads}{https://bitbucket.org/nikratio/s3ql/downloads} +Download S3QL from \url{https://bitbucket.org/nikratio/s3ql/downloads} \item {} Unpack it into a folder of your choice \item {} -Run \code{python3 setup.py build\_ext -{-}inplace} to build S3QL. +Run \sphinxcode{python3 setup.py build\_ext -{-}inplace} to build S3QL. \item {} -Run \code{python3 -m pytest tests/} to run a self-test. If this fails, ask +Run \sphinxcode{python3 -m pytest tests/} to run a self-test. If this fails, ask for help on the \href{http://groups.google.com/group/s3ql}{mailing list} or report a bug in the \href{https://bitbucket.org/nikratio/s3ql/issues}{issue tracker}. @@ -345,16 +369,16 @@ Now you have three options: \begin{itemize} \item {} -You can run the S3QL commands from the \code{bin/} directory. +You can run the S3QL commands from the \sphinxcode{bin/} directory. \item {} You can install S3QL system-wide for all users. To do that, you -have to run \code{sudo python3 setup.py install}. +have to run \sphinxcode{sudo python3 setup.py install}. \item {} -You can install S3QL into \code{\textasciitilde{}/.local} by executing \code{python3 +You can install S3QL into \sphinxcode{\textasciitilde{}/.local} by executing \sphinxcode{python3 setup.py install -{-}user}. In this case you should make sure that -\code{\textasciitilde{}/.local/bin} is in your \code{\$PATH} environment variable. +\sphinxcode{\textasciitilde{}/.local/bin} is in your \sphinxcode{\$PATH} environment variable. \end{itemize} @@ -387,7 +411,7 @@ with functionality. For example, any use of functions that are scheduled for deprecation in future Python version will cause tests to fail. If you would rather just check for functionality, you can delete -the \code{MANIFEST.in} file. In that case, the build system will +the \sphinxcode{MANIFEST.in} file. In that case, the build system will behave as it does for a regular release. The HTML and PDF documentation can be generated with @@ -408,7 +432,7 @@ By default, tests requiring a connection to a remote storage backend are skipped. If you would like to run these tests too (which is always a good idea), you have to create additional entries in your -\code{\textasciitilde{}/.s3ql/authinfo2} file that tell S3QL what server and credentials to +\sphinxcode{\textasciitilde{}/.s3ql/authinfo2} file that tell S3QL what server and credentials to use for these tests. These entries have the following form: \begin{Verbatim}[commandchars=\\\{\}] @@ -435,15 +459,15 @@ \PYG{l}{test\PYGZhy{}fs: gs://joes\PYGZhy{}gs\PYGZhy{}bucket/s3ql\PYGZus{}tests/} \end{Verbatim} -On the next run of \code{runtest.py} (or \code{py.test} when using the +On the next run of \sphinxcode{runtest.py} (or \sphinxcode{py.test} when using the development version), the additional tests will be run. If the tests are still skipped, you can get more information about why tests are -being skipped by passing the \code{-rs} argument to -\code{runtest.py}/\code{py.test}. +being skipped by passing the \sphinxcode{-rs} argument to +\sphinxcode{runtest.py}/\sphinxcode{py.test}. \chapter{Storage Backends} -\label{backends:sphinx}\label{backends:id1}\label{backends:storage-backends}\label{backends::doc} +\label{backends:sphinx}\label{backends::doc}\label{backends:id1}\label{backends:storage-backends} S3QL supports different \emph{backends} to store data at different service providers and using different protocols. A \emph{storage url} specifies a backend together with some backend-specific information and uniquely @@ -451,12 +475,12 @@ the backend and is described for every backend below. Furthermore, every S3QL commands that accepts a storage url also -accepts a \code{-{-}backend-options} parameter than can be used to +accepts a \sphinxcode{-{-}backend-options} parameter than can be used to pass backend-specific options to the backend module. The available options are documented with the respective backends below. -All storage backends respect the \code{http\_proxy} (for plain HTTP -connections) and \code{https\_proxy} (for SSL connections) +All storage backends respect the \sphinxcode{http\_proxy} (for plain HTTP +connections) and \sphinxcode{https\_proxy} (for SSL connections) environment variables. \begin{notice}{note}{Note:} @@ -464,8 +488,8 @@ you can e.g. copy the data stored by the local backend into Amazon S3 using some non-S3QL tool and then access it with S3QL's S3 backend. If you want to copy file systems from one backend to -another, you need to use the \code{clone\_fs.py} script (from the -\code{contrib} directory in the S3QL tarball). +another, you need to use the \sphinxcode{clone\_fs.py} script (from the +\sphinxcode{contrib} directory in the S3QL tarball). \end{notice} @@ -488,13 +512,13 @@ password. \item {} -Use OAuth2 authentication. In this case you need to use \code{oauth2} +Use OAuth2 authentication. In this case you need to use \sphinxcode{oauth2} as the backend login, and a valid OAuth2 refresh token as the backend password. To obtain a refresh token, you can use the -{\hyperref[man/oauth_client:oauth-client]{\emph{s3ql\_oauth\_client}}} program. It will instruct +{\hyperref[man/oauth_client:oauth\string-client]{\sphinxcrossref{\DUrole{std,std-ref}{s3ql\_oauth\_client}}}} program. It will instruct you to open a specific URL in your browser, enter a code and authenticate with your Google account. Once this procedure is -complete, {\hyperref[man/oauth_client:oauth-client]{\emph{s3ql\_oauth\_client}}} will print out +complete, {\hyperref[man/oauth_client:oauth\string-client]{\sphinxcrossref{\DUrole{std,std-ref}{s3ql\_oauth\_client}}}} will print out the refresh token. Note that you need to do this procedure only once, the refresh token will remain valid until you explicitly revoke it. @@ -518,7 +542,7 @@ \index{gs\_backend command line option!no-ssl}\index{no-ssl!gs\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-gs_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +\phantomsection\label{backends:cmdoption-gs_backend-arg-no-ssl}\pysigline{\sphinxbfcode{no-ssl}\sphinxcode{}} Disable encrypted (https) connections and use plain HTTP instead. \end{fulllineitems} @@ -526,10 +550,10 @@ \index{gs\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!gs\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-gs_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +\phantomsection\label{backends:cmdoption-gs_backend-arg-ssl-ca-path}\pysigline{\sphinxbfcode{ssl-ca-path}\sphinxcode{=\textless{}path\textgreater{}}} Instead of using the system's default certificate store, validate the server certificate against the specified CA -certificates. \code{\textless{}path\textgreater{}} may be either a file containing +certificates. \sphinxcode{\textless{}path\textgreater{}} may be either a file containing multiple certificates, or a directory containing one certificate per file. @@ -538,7 +562,7 @@ \index{gs\_backend command line option!tcp-timeout}\index{tcp-timeout!gs\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-gs_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +\phantomsection\label{backends:cmdoption-gs_backend-arg-tcp-timeout}\pysigline{\sphinxbfcode{tcp-timeout}\sphinxcode{}} Specifies the timeout used for TCP connections. If no data can be exchanged with the remote server for longer than this period, the TCP connection is closed and re-established (default: 20 seconds). @@ -548,7 +572,7 @@ \section{Amazon S3} -\label{backends:amazon-s3}\label{backends:google-storage-manager} +\label{backends:google-storage-manager}\label{backends:amazon-s3} \href{http://aws.amazon.com/s3}{Amazon S3} is the online storage service offered by \href{http://aws.amazon.com/}{Amazon Web Services (AWS)}. To use the S3 backend, you first need to sign up for an AWS account. The @@ -558,7 +582,7 @@ Console}. For best performance, it is recommend to create the bucket in the geographically closest storage region, but not the US Standard region -(see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for the reason). +(see {\hyperref[durability:durability]{\sphinxcrossref{\DUrole{std,std-ref}{Important Rules to Avoid Losing Data}}}} for the reason). The storage URL for accessing S3 buckets in S3QL has the form @@ -587,7 +611,7 @@ \index{s3\_backend command line option!no-ssl}\index{no-ssl!s3\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +\phantomsection\label{backends:cmdoption-s3_backend-arg-no-ssl}\pysigline{\sphinxbfcode{no-ssl}\sphinxcode{}} Disable encrypted (https) connections and use plain HTTP instead. \end{fulllineitems} @@ -595,10 +619,10 @@ \index{s3\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!s3\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +\phantomsection\label{backends:cmdoption-s3_backend-arg-ssl-ca-path}\pysigline{\sphinxbfcode{ssl-ca-path}\sphinxcode{=\textless{}path\textgreater{}}} Instead of using the system's default certificate store, validate the server certificate against the specified CA -certificates. \code{\textless{}path\textgreater{}} may be either a file containing +certificates. \sphinxcode{\textless{}path\textgreater{}} may be either a file containing multiple certificates, or a directory containing one certificate per file. @@ -607,7 +631,7 @@ \index{s3\_backend command line option!tcp-timeout}\index{tcp-timeout!s3\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +\phantomsection\label{backends:cmdoption-s3_backend-arg-tcp-timeout}\pysigline{\sphinxbfcode{tcp-timeout}\sphinxcode{}} Specifies the timeout used for TCP connections. If no data can be exchanged with the remote server for longer than this period, the TCP connection is closed and re-established (default: 20 seconds). @@ -617,7 +641,7 @@ \index{s3\_backend command line option!sse}\index{sse!s3\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3_backend-arg-sse}\pysigline{\bfcode{sse}\code{}} +\phantomsection\label{backends:cmdoption-s3_backend-arg-sse}\pysigline{\sphinxbfcode{sse}\sphinxcode{}} Enable server side encryption. Both costs \& benefits of S3 server side encryption are probably rather small, and this option does \emph{not} affect any client side encryption performed by S3QL itself. @@ -627,7 +651,7 @@ \index{s3\_backend command line option!ia}\index{ia!s3\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3_backend-arg-ia}\pysigline{\bfcode{ia}\code{}} +\phantomsection\label{backends:cmdoption-s3_backend-arg-ia}\pysigline{\sphinxbfcode{ia}\sphinxcode{}} Use infrequent access storage class for new objects. \end{fulllineitems} @@ -635,15 +659,15 @@ \index{s3\_backend command line option!rrs}\index{rrs!s3\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3_backend-arg-rrs}\pysigline{\bfcode{rrs}\code{}} +\phantomsection\label{backends:cmdoption-s3_backend-arg-rrs}\pysigline{\sphinxbfcode{rrs}\sphinxcode{}} Enable reduced redundancy storage for newly created objects (overwrites the \emph{ia} option). When enabling this option, it is strongly recommended to -periodically run {\hyperref[fsck:s3ql-verify]{\emph{s3ql\_verify}}}, because objects +periodically run {\hyperref[fsck:s3ql\string-verify]{\sphinxcrossref{\DUrole{std,std-ref}{s3ql\_verify}}}}, because objects that are lost by the storage backend may cause subsequent data loss even later in time due to the data de-duplication feature of S3QL (see -{\hyperref[durability:backend-reliability]{\emph{Data Durability}}} for details). +{\hyperref[durability:backend\string-reliability]{\sphinxcrossref{\DUrole{std,std-ref}{Data Durability}}}} for details). \end{fulllineitems} @@ -663,13 +687,13 @@ \PYG{l}{swift://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} \end{Verbatim} -for keystore (v2) authentication, the storage URL is +for Keystone (v2) authentication, the storage URL is \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{swiftks://}\PYG{n+nv}{\PYGZlt{}hostname\PYGZgt{}}\PYG{g+ge}{[:\PYGZlt{}port\PYGZgt{}]}\PYG{l}{/}\PYG{n+nv}{\PYGZlt{}region\PYGZgt{}}\PYG{l}{:}\PYG{n+nv}{\PYGZlt{}container\PYGZgt{}}\PYG{g+ge}{[/\PYGZlt{}prefix\PYGZgt{}]} \end{Verbatim} -Note that when using keystore authentication, you can (and have to) +Note that when using Keystone authentication, you can (and have to) specify the storage region of the container as well. In both cases, \emph{hostname} name should be the name of the @@ -681,9 +705,9 @@ When using legacy authentication, the backend login and password correspond to the OpenStack username and API Access Key. When using -keystore authentication, the backend password is your regular +Keystone authentication, the backend password is your regular OpenStack password and the backend login combines you OpenStack -username and tenant name in the form \code{\textless{}tenant\textgreater{}:\textless{}user\textgreater{}}. If no tenant +username and tenant name in the form \sphinxcode{\textless{}tenant\textgreater{}:\textless{}user\textgreater{}}. If no tenant is required, the OpenStack username alone may be used as backend login. @@ -691,7 +715,7 @@ \index{swift\_backend command line option!no-ssl}\index{no-ssl!swift\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-swift_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +\phantomsection\label{backends:cmdoption-swift_backend-arg-no-ssl}\pysigline{\sphinxbfcode{no-ssl}\sphinxcode{}} Use plain HTTP to connect to the authentication server. This option does not directly affect the connection to the storage server. Whether HTTPS or plain HTTP is used to connect to the @@ -702,10 +726,10 @@ \index{swift\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!swift\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-swift_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +\phantomsection\label{backends:cmdoption-swift_backend-arg-ssl-ca-path}\pysigline{\sphinxbfcode{ssl-ca-path}\sphinxcode{=\textless{}path\textgreater{}}} Instead of using the system's default certificate store, validate the server certificate against the specified CA -certificates. \code{\textless{}path\textgreater{}} may be either a file containing +certificates. \sphinxcode{\textless{}path\textgreater{}} may be either a file containing multiple certificates, or a directory containing one certificate per file. @@ -714,7 +738,7 @@ \index{swift\_backend command line option!tcp-timeout}\index{tcp-timeout!swift\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-swift_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +\phantomsection\label{backends:cmdoption-swift_backend-arg-tcp-timeout}\pysigline{\sphinxbfcode{tcp-timeout}\sphinxcode{}} Specifies the timeout used for TCP connections. If no data can be exchanged with the remote server for longer than this period, the TCP connection is closed and re-established (default: 20 seconds). @@ -724,8 +748,8 @@ \index{swift\_backend command line option!disable-expect100}\index{disable-expect100!swift\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-swift_backend-arg-disable-expect100}\pysigline{\bfcode{disable-expect100}\code{}} -If this option is specified, S3QL does not use the \code{Expect: +\phantomsection\label{backends:cmdoption-swift_backend-arg-disable-expect100}\pysigline{\sphinxbfcode{disable-expect100}\sphinxcode{}} +If this option is specified, S3QL does not use the \sphinxcode{Expect: continue} header (cf. \href{http://tools.ietf.org/html/rfc2616\#section-8.2.3}{RFC2616, section 8.2.3}) when uploading data to the server. This can be used to work around broken storage servers that don't fully support HTTP 1.1, but may decrease @@ -734,14 +758,34 @@ \end{fulllineitems} +\index{swift\_backend command line option!no-feature-detection}\index{no-feature-detection!swift\_backend command line option} + +\begin{fulllineitems} +\phantomsection\label{backends:cmdoption-swift_backend-arg-no-feature-detection}\pysigline{\sphinxbfcode{no-feature-detection}\sphinxcode{}} +If this option is specified, S3QL does not try to dynamically detect +advanced features of the Swift backend. In this case S3QL can only +use the least common denominator of supported Swift versions and +configurations. + +\end{fulllineitems} + + +\begin{notice}{note}{Note:} +The Swift API unfortunately lacks a number of features that S3QL +normally makes use of. S3QL works around these deficiencies as much +as possible. However, this means that storing data using the Swift +backend generally requires more network round-trips and transfer +volume than the other backends. Also, S3QL requires Swift storage +servers to provide immediate consistency for newly created objects. +\end{notice} \section{Rackspace CloudFiles} -\label{backends:rackspace-cloudfiles}\label{backends:swift} +\label{backends:rackspace-cloudfiles} \href{http://www.rackspace.com/}{Rackspace} CloudFiles uses \href{http://www.openstack.org/}{OpenStack} internally, so it is possible to just use the OpenStack/Swift backend (see above) with -\code{auth.api.rackspacecloud.com} as the host name. For convenince, -there is also a special \code{rackspace} backend that uses a storage URL +\sphinxcode{auth.api.rackspacecloud.com} as the host name. For convenince, +there is also a special \sphinxcode{rackspace} backend that uses a storage URL of the form \begin{Verbatim}[commandchars=\\\{\}] @@ -758,18 +802,7 @@ topmost menu bar). The Rackspace backend accepts the same backend options as the -{\hyperref[backends:openstack-backend]{\emph{OpenStack backend}}}. - -\begin{notice}{note}{Note:} -As of January 2012, Rackspace does not give any durability or -consistency guarantees (see {\hyperref[durability:durability]{\emph{Important Rules to Avoid Losing Data}}} for why this is -important). However, Rackspace support agents seem prone to claim -very high guarantees. Unless explicitly backed by their terms of -service, any such statement should thus be viewed with -suspicion. S3QL developers have also \href{http://www.rath.org/Tales\%20from\%20the\%20Rackspace\%20Support}{repeatedly experienced} -similar issues with the credibility and competence of the Rackspace -support. -\end{notice} +{\hyperref[backends:openstack\string-backend]{\sphinxcrossref{\DUrole{std,std-ref}{OpenStack backend}}}}. \section{S3 compatible} @@ -790,7 +823,7 @@ \index{s3c\_backend command line option!no-ssl}\index{no-ssl!s3c\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3c_backend-arg-no-ssl}\pysigline{\bfcode{no-ssl}\code{}} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-no-ssl}\pysigline{\sphinxbfcode{no-ssl}\sphinxcode{}} Disable encrypted (https) connections and use plain HTTP instead. \end{fulllineitems} @@ -798,10 +831,10 @@ \index{s3c\_backend command line option!ssl-ca-path=\textless{}path\textgreater{}}\index{ssl-ca-path=\textless{}path\textgreater{}!s3c\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3c_backend-arg-ssl-ca-path}\pysigline{\bfcode{ssl-ca-path}\code{=\textless{}path\textgreater{}}} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-ssl-ca-path}\pysigline{\sphinxbfcode{ssl-ca-path}\sphinxcode{=\textless{}path\textgreater{}}} Instead of using the system's default certificate store, validate the server certificate against the specified CA -certificates. \code{\textless{}path\textgreater{}} may be either a file containing +certificates. \sphinxcode{\textless{}path\textgreater{}} may be either a file containing multiple certificates, or a directory containing one certificate per file. @@ -810,7 +843,7 @@ \index{s3c\_backend command line option!tcp-timeout}\index{tcp-timeout!s3c\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3c_backend-arg-tcp-timeout}\pysigline{\bfcode{tcp-timeout}\code{}} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-tcp-timeout}\pysigline{\sphinxbfcode{tcp-timeout}\sphinxcode{}} Specifies the timeout used for TCP connections. If no data can be exchanged with the remote server for longer than this period, the TCP connection is closed and re-established (default: 20 seconds). @@ -820,8 +853,8 @@ \index{s3c\_backend command line option!disable-expect100}\index{disable-expect100!s3c\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3c_backend-arg-disable-expect100}\pysigline{\bfcode{disable-expect100}\code{}} -If this option is specified, S3QL does not use the \code{Expect: +\phantomsection\label{backends:cmdoption-s3c_backend-arg-disable-expect100}\pysigline{\sphinxbfcode{disable-expect100}\sphinxcode{}} +If this option is specified, S3QL does not use the \sphinxcode{Expect: continue} header (cf. \href{http://tools.ietf.org/html/rfc2616\#section-8.2.3}{RFC2616, section 8.2.3}) when uploading data to the server. This can be used to work around broken storage servers that don't fully support HTTP 1.1, but may decrease @@ -833,14 +866,14 @@ \index{s3c\_backend command line option!dumb-copy}\index{dumb-copy!s3c\_backend command line option} \begin{fulllineitems} -\phantomsection\label{backends:cmdoption-s3c_backend-arg-dumb-copy}\pysigline{\bfcode{dumb-copy}\code{}} +\phantomsection\label{backends:cmdoption-s3c_backend-arg-dumb-copy}\pysigline{\sphinxbfcode{dumb-copy}\sphinxcode{}} If this option is specified, S3QL assumes that a COPY request to the storage server has succeeded as soon as the server returns a -\code{200 OK} status. The \href{http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html}{S3 COPY API} specifies that the +\sphinxcode{200 OK} status. The \href{http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectCOPY.html}{S3 COPY API} specifies that the storage server may still return an error in the request body (see the \href{https://doc.s3.amazonaws.com/proposals/copy.html}{copy proposal} for the rationale), so this option should only be used if you are certain that your storage -server only returns \code{200 OK} when the copy operation has been +server only returns \sphinxcode{200 OK} when the copy operation has been completely and successfully carried out. Using this option may be neccessary if your storage server does not return a valid response body for a succesfull copy operation. @@ -850,7 +883,7 @@ \section{Local} -\label{backends:id6}\label{backends:local} +\label{backends:local}\label{backends:id6} S3QL is also able to store its data on the local file system. This can be used to backup data on external media, or to access external services that S3QL can not talk to directly (e.g., it is possible to @@ -865,18 +898,18 @@ \end{Verbatim} Note that you have to write three consecutive slashes to specify an -absolute path, e.g. \code{local:///var/archive}. Also, relative paths will +absolute path, e.g. \sphinxcode{local:///var/archive}. Also, relative paths will automatically be converted to absolute paths before the authentication -file (see {\hyperref[authinfo:authinfo]{\emph{Storing Authentication Information}}}) is read, i.e. if you are in the -\code{/home/john} directory and try to mount \code{local://s3ql}, the +file (see {\hyperref[authinfo:authinfo]{\sphinxcrossref{\DUrole{std,std-ref}{Storing Authentication Information}}}}) is read, i.e. if you are in the +\sphinxcode{/home/john} directory and try to mount \sphinxcode{local://s3ql}, the corresponding section in the authentication file must match the -storage url \code{local:///home/john/s3ql}. +storage url \sphinxcode{local:///home/john/s3ql}. The local backend does not accept any backend options. \chapter{Important Rules to Avoid Losing Data} -\label{durability:sshfs}\label{durability:durability}\label{durability:important-rules-to-avoid-losing-data}\label{durability::doc} +\label{durability:durability}\label{durability:sshfs}\label{durability::doc}\label{durability:important-rules-to-avoid-losing-data} Most S3QL backends store data in distributed storage systems. These systems differ from a traditional, local hard disk in several important ways. In order to avoid losing data, this section should be @@ -924,14 +957,14 @@ \item {} When mounting the same file system on different computers (or on -the same computer but with different \code{-{-}cachedir} directories), +the same computer but with different \sphinxcode{-{-}cachedir} directories), the time that passes between the first and second of invocation of -\textbf{mount.s3ql} must be at least as long as your storage +\textbf{\texttt{mount.s3ql}} must be at least as long as your storage service's consistency window. If your storage service offers immediate consistency, you do not need to wait at all. \item {} -Before running \textbf{fsck.s3ql} or \textbf{s3qladm}, the file system +Before running \textbf{\texttt{fsck.s3ql}} or \textbf{\texttt{s3qladm}}, the file system must have been left untouched for the length of the consistency window. If your storage service offers immediate consistency, you do not need to wait at all. @@ -951,26 +984,26 @@ not included, or if you need more reliable information, check with your storage provider. -\begin{tabulary}{\linewidth}{|L|L|} +\noindent\begin{tabulary}{\linewidth}{|L|L|} \hline \textsf{\relax Storage Provider -} & \textsf{\relax +\unskip}\relax &\textsf{\relax Consistency Window -}\\ +\unskip}\relax \\ \hline Amazon S3 in the US standard region - & +& No guarantees \\ \hline Amazon S3 in other regions - & +& Immediate \\ \hline Google Storage - & +& Immediate \\ \hline\end{tabulary} @@ -1014,7 +1047,7 @@ status (which S3QL necessarily has to consider as current) and thereby permanently override the newer version (even though this change may not become immediately visible either). S3QL uses several techniques -to reduce the likelihood of this to happen (see {\hyperref[impl_details:impl-details]{\emph{Implementation Details}}} +to reduce the likelihood of this to happen (see {\hyperref[impl_details:impl\string-details]{\sphinxcrossref{\DUrole{std,std-ref}{Implementation Details}}}} for more information on this), but without support from the storage service, the possibility cannot be eliminated completely. @@ -1099,7 +1132,7 @@ \item {} The storage service loses the data blocks of this file. As long as you -do not access the file or run \textbf{fsck.s3ql}, S3QL is not +do not access the file or run \textbf{\texttt{fsck.s3ql}}, S3QL is not aware that the data has been lost by the storage service. \item {} @@ -1120,29 +1153,29 @@ \end{enumerate} -For some storage services, \textbf{fsck.s3ql} can mitigate this -effect. When \textbf{fsck.s3ql} runs, it asks the storage service +For some storage services, \textbf{\texttt{fsck.s3ql}} can mitigate this +effect. When \textbf{\texttt{fsck.s3ql}} runs, it asks the storage service for a list of all stored objects. If objects are missing, it can then mark the damaged files and prevent the problem from spreading forwards in time. Figuratively speaking, this establishes a ``checkpoint'': data -loss that occurred before running \textbf{fsck.s3ql} can not affect +loss that occurred before running \textbf{\texttt{fsck.s3ql}} can not affect any file system operations that are performed after the check. Unfortunately, many storage services only ``discover'' that objects are missing or broken when the object actually needs to be retrieved. In -this case, \textbf{fsck.s3ql} will not learn anything by just +this case, \textbf{\texttt{fsck.s3ql}} will not learn anything by just querying the list of objects. This effect can be mitigated to some degree by using the -\textbf{s3ql\_verify} command in additon to -\textbf{fsck.s3ql}. \textbf{s3ql\_verify} asks the storage service +\textbf{\texttt{s3ql\_verify}} command in additon to +\textbf{\texttt{fsck.s3ql}}. \textbf{\texttt{s3ql\_verify}} asks the storage service to look up every stored object and may therefore take much longer than -running \textbf{fsck.s3ql}, but can also offer a much stronger +running \textbf{\texttt{fsck.s3ql}}, but can also offer a much stronger assurance that no data has been lost by the storage service. To ``recover'' from damaged storage objects in the backend, the damaged -objects found by \textbf{s3ql\_verify} have to be explicitly deleted -(so that a successive \textbf{fsck.s3ql} is able detect them as +objects found by \textbf{\texttt{s3ql\_verify}} have to be explicitly deleted +(so that a successive \textbf{\texttt{fsck.s3ql}} is able detect them as missing, correct the file system metadata, and move any affected files -to \code{lost+found}). This procedure is currently not automated, so +to \sphinxcode{lost+found}). This procedure is currently not automated, so it is generally a good idea to choose a storage service where the expected data durability is high enough so that the possibility of a lost object (and thus the need to run any full checks) can be @@ -1151,7 +1184,7 @@ \chapter{File System Creation} \label{mkfs:file-system-creation}\label{mkfs::doc} -A S3QL file system is created with the \textbf{mkfs.s3ql} command. It has the +A S3QL file system is created with the \textbf{\texttt{mkfs.s3ql}} command. It has the following syntax: \begin{Verbatim}[commandchars=\\\{\}] @@ -1163,19 +1196,19 @@ \begin{optionlist}{3cm} \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}backend-options \textless{}options\textgreater{}] @@ -1196,10 +1229,10 @@ \end{optionlist} \end{quote} -Unless you have specified the \code{-{-}plain} option, -\textbf{mkfs.s3ql} will ask you to enter an encryption +Unless you have specified the \sphinxcode{-{-}plain} option, +\textbf{\texttt{mkfs.s3ql}} will ask you to enter an encryption password. This password will \emph{not} be read from an authentication file -specified with the \code{-{-}authfile} option to prevent accidental +specified with the \sphinxcode{-{-}authfile} option to prevent accidental creation of an encrypted file system. Note that: @@ -1208,29 +1241,29 @@ All data that is stored under the given storage url is assumed to managed exclusively by S3QL. Trying to manually save additional objects (or remove or manipulate existing objects) will lead to file -system corruption, and \textbf{fsck.s3ql} may delete objects that +system corruption, and \textbf{\texttt{fsck.s3ql}} may delete objects that do not belong to the file system. \item {} With most storage backends, slashes in the storage url prefix do not have special meaning. For example, the storage urls -\code{s3://mybucket/myprefix/} and \code{s3://mybucket/myprefix} are -distinct. In the first case, the prefix is \code{myprefix/}, while in -the second it is \code{myprefix}. +\sphinxcode{s3://mybucket/myprefix/} and \sphinxcode{s3://mybucket/myprefix} are +distinct. In the first case, the prefix is \sphinxcode{myprefix/}, while in +the second it is \sphinxcode{myprefix}. \item {} S3QL file systems can not be ``stacked'', i.e. you cannot have one -file system stored at \code{s3://bucketname/outerprefix} and a second -one at \code{s3://bucketname/outerprefix/innerprefix}. +file system stored at \sphinxcode{s3://bucketname/outerprefix} and a second +one at \sphinxcode{s3://bucketname/outerprefix/innerprefix}. \end{itemize} \chapter{Managing File Systems} \label{adm:managing-file-systems}\label{adm::doc} -The \code{s3qladm} command performs various operations on \emph{unmounted} S3QL +The \sphinxcode{s3qladm} command performs various operations on \emph{unmounted} S3QL file systems. The file system \emph{must not be mounted} when using -\code{s3qladm} or things will go wrong badly. +\sphinxcode{s3qladm} or things will go wrong badly. The syntax is @@ -1238,10 +1271,10 @@ \PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage\PYGZhy{}url\PYGZgt{}} \end{Verbatim} -where \code{action} may be either of \textbf{passphrase}, -\textbf{upgrade}, \textbf{clear} or \textbf{download-metadata}. +where \sphinxcode{action} may be either of \textbf{\texttt{passphrase}}, +\textbf{\texttt{upgrade}}, \textbf{\texttt{clear}} or \textbf{\texttt{download-metadata}}. -The \textbf{s3qladm} accepts the following general options, no +The \textbf{\texttt{s3qladm}} accepts the following general options, no matter what specific action is being invoked: \begin{quote} \begin{optionlist}{3cm} @@ -1249,41 +1282,41 @@ Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}log \textless{}target\textgreater{}] -Destination for log messages. Specify \code{none} for -standard output or \code{syslog} for the system logging +Destination for log messages. Specify \sphinxcode{none} for +standard output or \sphinxcode{syslog} for the system logging daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: -\code{None} +\sphinxcode{None} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}version] just print program version and exit \end{optionlist} \end{quote} -Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments +Hint: run \sphinxcode{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments that the different actions take. \section{Changing the Passphrase} \label{adm:changing-the-passphrase} -To change the passphrase of a file system, use the \code{passphrase} +To change the passphrase of a file system, use the \sphinxcode{passphrase} subcommand: \begin{Verbatim}[commandchars=\\\{\}] @@ -1335,19 +1368,19 @@ will give you a list of the available metadata backups and allow you to download them. This will create two new files in the current -directory, ending in \code{.db} and \code{.params}. To actually use the -downloaded backup, you need to move these files into the \code{\textasciitilde{}/.s3ql/} -directory and run \code{fsck.s3ql}. +directory, ending in \sphinxcode{.db} and \sphinxcode{.params}. To actually use the +downloaded backup, you need to move these files into the \sphinxcode{\textasciitilde{}/.s3ql/} +directory and run \sphinxcode{fsck.s3ql}. \begin{notice}{warning}{Warning:} You should probably not use this functionality without having asked -for help on the mailing list first (see {\hyperref[resources:resources]{\emph{Further Resources / Getting Help}}}). +for help on the mailing list first (see {\hyperref[resources:resources]{\sphinxcrossref{\DUrole{std,std-ref}{Further Resources / Getting Help}}}}). \end{notice} \chapter{Mounting} \label{mount:mounting}\label{mount::doc} -A S3QL file system is mounted with the \textbf{mount.s3ql} +A S3QL file system is mounted with the \textbf{\texttt{mount.s3ql}} command. It has the following syntax: \begin{Verbatim}[commandchars=\\\{\}] @@ -1364,27 +1397,27 @@ \begin{quote} \begin{optionlist}{3cm} \item [-{-}log \textless{}target\textgreater{}] -Destination for log messages. Specify \code{none} for -standard output or \code{syslog} for the system logging +Destination for log messages. Specify \sphinxcode{none} for +standard output or \sphinxcode{syslog} for the system logging daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: -\code{\textasciitilde{}/.s3ql/mount.log} +\sphinxcode{\textasciitilde{}/.s3ql/mount.log} \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}backend-options \textless{}options\textgreater{}] @@ -1399,18 +1432,18 @@ autodetect). Each cache entry requires one file descriptor, so if you increase this number you have to make sure that your process file descriptor limit (as -set with \code{ulimit -n}) is high enough (at least the +set with \sphinxcode{ulimit -n}) is high enough (at least the number of cache entries + 100). \item [-{-}allow-other] -Normally, only the user who called \code{mount.s3ql} can +Normally, only the user who called \sphinxcode{mount.s3ql} can access the mount point. This user then also has full access to it, independent of individual file -permissions. If the \code{-{-}allow-other} option is +permissions. If the \sphinxcode{-{-}allow-other} option is specified, other users can access the mount point as well and individual file permissions are taken into account for all users. \item [-{-}allow-root] -Like \code{-{-}allow-other}, but restrict access to the +Like \sphinxcode{-{-}allow-other}, but restrict access to the mounting user and the root user. \item [-{-}fg] Do not daemonize, stay in foreground @@ -1420,9 +1453,9 @@ \item [-{-}compress \textless{}algorithm-lvl\textgreater{}] Compression algorithm and compression level to use when storing new data. \emph{algorithm} may be any of -\code{lzma}, \code{bzip2}, \code{zlib}, or none. \emph{lvl} may be any +\sphinxcode{lzma}, \sphinxcode{bzip2}, \sphinxcode{zlib}, or none. \emph{lvl} may be any integer from 0 (fastest) to 9 (slowest). Default: -\code{lzma-6} +\sphinxcode{lzma-6} \item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. @@ -1438,8 +1471,8 @@ \section{Permission Checking} \label{mount:permission-checking} -If the file system is mounted with neither the \code{allow-root} -nor \code{allow-other} option, the mounting user has full +If the file system is mounted with neither the \sphinxcode{allow-root} +nor \sphinxcode{allow-other} option, the mounting user has full permissions on the S3QL file system (he is effectively root). If one (or both) of the options is used, standard unix permission checks apply, i.e. only the real root user has full access and all other @@ -1463,7 +1496,7 @@ To find the optimal algorithm and number of parallel compression threads for your system, S3QL ships with a program called -\code{benchmark.py} in the \code{contrib} directory. You should run this program +\sphinxcode{benchmark.py} in the \sphinxcode{contrib} directory. You should run this program on a file that has a size that is roughly equal to the block size of your file system and has similar contents. It will then determine the compression speeds for the different algorithms and the upload speeds @@ -1471,7 +1504,7 @@ fast enough to saturate your network connection. Obviously you should make sure that there is little other system load -when you run \code{benchmark.py} (i.e., don't compile software or encode +when you run \sphinxcode{benchmark.py} (i.e., don't compile software or encode videos at the same time). @@ -1485,19 +1518,19 @@ \subsection{Maximum Number of Cache Entries} \label{mount:maximum-number-of-cache-entries} The maximum size of the cache can be configured with the -\code{-{-}cachesize} option. In addition to that, the maximum number +\sphinxcode{-{-}cachesize} option. In addition to that, the maximum number of objects in the cache is limited by the -\code{-{-}max-cache-entries} option, so it is possible that the cache +\sphinxcode{-{-}max-cache-entries} option, so it is possible that the cache does not grow up to the maximum cache size because the maximum number of cache elements has been reached. The reason for this limit is that each cache entry requires one open file descriptor, and Linux distributions usually limit the total number of file descriptors per process to about a thousand. -If you specify a value for \code{-{-}max-cache-entries}, you should +If you specify a value for \sphinxcode{-{-}max-cache-entries}, you should therefore make sure to also configure your system to increase the maximum number of open file handles. This can be done temporarily with -the \textbf{ulimit -n} command. The method to permanently change this limit +the \textbf{\texttt{ulimit -n}} command. The method to permanently change this limit system-wide depends on your distribution. @@ -1516,7 +1549,7 @@ \label{mount:failure-modes} Once an S3QL file system has been mounted, there is a multitude of problems that can occur when communicating with the remote -server. Generally, \textbf{mount.s3ql} always tries to keep the file +server. Generally, \textbf{\texttt{mount.s3ql}} always tries to keep the file system as accessible as possible under the circumstances. That means that if network connectivity is lost, data can still be written as long as there is space in the local cache. Attempts to read data not @@ -1524,33 +1557,33 @@ re-established. If any sort of data corruption is detected, the file system will switch to read-only mode. Attempting to read files that are affected by the corruption will return an input/output error -(\emph{errno} set to \code{EIO}). +(\emph{errno} set to \sphinxcode{EIO}). -In case of other unexpected or fatal problems, \textbf{mount.s3ql} +In case of other unexpected or fatal problems, \textbf{\texttt{mount.s3ql}} terminates, but does not unmount the file system. Any attempt to access the mountpoint will result in a ``Transport endpoint not -connected'' error (\emph{errno} set to \code{ESHUTDOWN}). This ensures that a -mountpoint whose \textbf{mount.s3ql} process has terminated can not +connected'' error (\emph{errno} set to \sphinxcode{ESHUTDOWN}). This ensures that a +mountpoint whose \textbf{\texttt{mount.s3ql}} process has terminated can not be confused with a mountpoint containing an empty file system (which would be fatal if e.g. the mountpoint is automatically mirrored). When this has happened, the mountpoint can be cleared by using the -\textbf{fusermount} command (provided by FUSE) with the \code{-u} +\textbf{\texttt{fusermount}} command (provided by FUSE) with the \sphinxcode{-u} parameter. -\textbf{mount.s3ql} will automatically try to re-establish the +\textbf{\texttt{mount.s3ql}} will automatically try to re-establish the connection to the server if network connectivity is lost, and retry sending a request when the connection is established but the remote server signals a temporary problem. These attempts will be made at increasing intervals for a period up to 24 hours, with retry intervals starting at 20 ms and increasing up to 5 minutes. After 24 hours, -\textbf{mount.s3ql} will give up and terminate, leaving the +\textbf{\texttt{mount.s3ql}} will give up and terminate, leaving the mountpoint inaccessible as described above. -Generally, \textbf{mount.s3ql} will also emit log messages for any +Generally, \textbf{\texttt{mount.s3ql}} will also emit log messages for any unusual conditions that it encounters. The destination for these -messages can be set with the \code{-{-}log} parameter. It is highly +messages can be set with the \sphinxcode{-{-}log} parameter. It is highly recommended to periodically check these logs, for example with a tool -like \href{http://sourceforge.net/projects/logcheck/}{logcheck}. Many potential issues that \textbf{mount.s3ql} may +like \href{http://sourceforge.net/projects/logcheck/}{logcheck}. Many potential issues that \textbf{\texttt{mount.s3ql}} may encounter do not justify restricting access to the file system, but should nevertheless be investigated if they occur. Checking the log messages is the only way to find out about them. @@ -1560,24 +1593,24 @@ \label{mount:automatic-mounting}\label{mount:logcheck} If you want to mount and umount an S3QL file system automatically at system startup and shutdown, you should do so with a dedicated S3QL -init job (instead of using \code{/etc/fstab}. When using systemd, -\textbf{mount.s3ql} can be run as a service of type \code{notify}. +init job (instead of using \sphinxcode{/etc/fstab}. When using systemd, +\textbf{\texttt{mount.s3ql}} can be run as a service of type \sphinxcode{notify}. \begin{notice}{note}{Note:} In principle, it is also possible to automatically mount an S3QL -file system with an appropriate entry in \code{/etc/fstab}. However, +file system with an appropriate entry in \sphinxcode{/etc/fstab}. However, this is not recommended for several reasons: \begin{itemize} \item {} -file systems mounted in \code{/etc/fstab} will be unmounted with the -\textbf{umount} command, so your system will not wait until all data has +file systems mounted in \sphinxcode{/etc/fstab} will be unmounted with the +\textbf{\texttt{umount}} command, so your system will not wait until all data has been uploaded but shutdown (or restart) immediately (this is a FUSE limitation, see \href{https://bitbucket.org/nikratio/s3ql/issue/1/blocking-fusermount-and-umount}{issue \#1}). \item {} There is no way to tell the system that mounting S3QL requires a Python interpreter to be available, so it may attempt to run -\textbf{mount.s3ql} before it has mounted the volume containing +\textbf{\texttt{mount.s3ql}} before it has mounted the volume containing the Python interpreter. \item {} @@ -1590,51 +1623,51 @@ \chapter{Advanced S3QL Features} -\label{special:advanced-s3ql-features}\label{special::doc} +\label{special::doc}\label{special:advanced-s3ql-features} \section{Snapshotting and Copy-on-Write} \label{special:snapshotting-and-copy-on-write}\label{special:s3qlcp} -The command \code{s3qlcp} can be used to duplicate a directory tree without +The command \sphinxcode{s3qlcp} can be used to duplicate a directory tree without physically copying the file contents. This is made possible by the data de-duplication feature of S3QL. -The syntax of \code{s3qlcp} is: +The syntax of \sphinxcode{s3qlcp} is: \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{s3qlcp }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}src\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}target\PYGZgt{}} \end{Verbatim} -This will replicate the contents of the directory \code{\textless{}src\textgreater{}} in the -directory \code{\textless{}target\textgreater{}}. \code{\textless{}src\textgreater{}} has to be an existing directory and -\code{\textless{}target\textgreater{}} must not exist. Moreover, both directories have to be +This will replicate the contents of the directory \sphinxcode{\textless{}src\textgreater{}} in the +directory \sphinxcode{\textless{}target\textgreater{}}. \sphinxcode{\textless{}src\textgreater{}} has to be an existing directory and +\sphinxcode{\textless{}target\textgreater{}} must not exist. Moreover, both directories have to be within the same S3QL file system. The replication will not take any additional space. Only if one of directories is modified later on, the modified data will take additional storage space. -\code{s3qlcp} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root}) +\sphinxcode{s3qlcp} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or \sphinxcode{-{-}allow-root}) the root user. Note that: \begin{itemize} \item {} After the replication, both source and target directory will still -be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a -snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common -usage of \code{s3qlcp} is to regularly duplicate the same source -directory, say \code{documents}, to different target directories. For a +be completely ordinary directories. You can regard \sphinxcode{\textless{}src\textgreater{}} as a +snapshot of \sphinxcode{\textless{}target\textgreater{}} or vice versa. However, the most common +usage of \sphinxcode{s3qlcp} is to regularly duplicate the same source +directory, say \sphinxcode{documents}, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like \code{documents\_January} for the replication in -January, \code{documents\_February} for the replication in February etc. +named something like \sphinxcode{documents\_January} for the replication in +January, \sphinxcode{documents\_February} for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. \item {} Exactly the same effect could be achieved by an ordinary copy -program like \code{cp -a}. However, this procedure would be orders of -magnitude slower, because \code{cp} would have to read every file +program like \sphinxcode{cp -a}. However, this procedure would be orders of +magnitude slower, because \sphinxcode{cp} would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. @@ -1668,9 +1701,9 @@ \section{Getting Statistics} -\label{special:getting-statistics}\label{special:s3qlstat} +\label{special:s3qlstat}\label{special:getting-statistics} You can get more information about a mounted S3QL file system with the -\code{s3qlstat} command. It has the following syntax: +\sphinxcode{s3qlstat} command. It has the following syntax: \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{s3qlstat }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}} @@ -1693,24 +1726,24 @@ the total size after duplication, and the final size after de-duplication and compression. -\code{s3qlstat} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root}) +\sphinxcode{s3qlstat} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or \sphinxcode{-{-}allow-root}) the root user. -For a full list of available options, run \code{s3qlstat -{-}help}. +For a full list of available options, run \sphinxcode{s3qlstat -{-}help}. \section{Immutable Trees} -\label{special:s3qllock}\label{special:immutable-trees} -The command \textbf{s3qllock} can be used to make a directory tree +\label{special:immutable-trees}\label{special:s3qllock} +The command \textbf{\texttt{s3qllock}} can be used to make a directory tree immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get -rid of an immutable tree is to use the \textbf{s3qlrm} command (see +rid of an immutable tree is to use the \textbf{\texttt{s3qlrm}} command (see below). For example, to make the directory tree beneath the directory -\code{2010-04-21} immutable, execute +\sphinxcode{2010-04-21} immutable, execute \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{s3qllock 2010\PYGZhy{}04\PYGZhy{}21} @@ -1736,7 +1769,7 @@ excluded, being able to change a backup after it has been made is generally not a good idea. A common S3QL use case is to keep the file system mounted at all times and periodically create backups with -\textbf{rsync -a}. This allows every user to recover her files from a +\textbf{\texttt{rsync -a}}. This allows every user to recover her files from a backup without having to call the system administrator. However, this also allows every user to accidentally change or delete files \emph{in} one of the old backups. @@ -1748,10 +1781,10 @@ \section{Fast Recursive Removal} -\label{special:s3qlrm}\label{special:fast-recursive-removal} -The \code{s3qlrm} command can be used to recursively delete files and -directories on an S3QL file system. Although \code{s3qlrm} is faster than -using e.g. \code{rm -r}, the main reason for its existence is that it +\label{special:fast-recursive-removal}\label{special:s3qlrm} +The \sphinxcode{s3qlrm} command can be used to recursively delete files and +directories on an S3QL file system. Although \sphinxcode{s3qlrm} is faster than +using e.g. \sphinxcode{rm -r}, the main reason for its existence is that it allows you to delete immutable trees as well. The syntax is rather simple: @@ -1765,15 +1798,15 @@ \section{Runtime Configuration} \label{special:s3qlctrl}\label{special:runtime-configuration} -The \code{s3qlctrl} can be used to control a mounted S3QL file system. Its +The \sphinxcode{s3qlctrl} can be used to control a mounted S3QL file system. Its syntax is \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ ...} \end{Verbatim} -\code{\textless{}mountpoint\textgreater{}} must be the location of a mounted S3QL file system. -For a list of valid options, run \code{s3qlctrl -{-}help}. \code{\textless{}action\textgreater{}} +\sphinxcode{\textless{}mountpoint\textgreater{}} must be the location of a mounted S3QL file system. +For a list of valid options, run \sphinxcode{s3qlctrl -{-}help}. \sphinxcode{\textless{}action\textgreater{}} may be either of: \begin{quote} \begin{quote}\begin{description} @@ -1795,7 +1828,7 @@ \chapter{Unmounting} -\label{umount:unmounting}\label{umount::doc} +\label{umount::doc}\label{umount:unmounting} To unmount an S3QL file system, use the command: \begin{Verbatim}[commandchars=\\\{\}] @@ -1804,27 +1837,27 @@ This will block until all data has been written to the backend. -Only the user who mounted the file system with \textbf{mount.s3ql} +Only the user who mounted the file system with \textbf{\texttt{mount.s3ql}} is able to unmount it again. If you are root and want to unmount an S3QL file system mounted by an ordinary user, you have to use the -\textbf{fusermount -u} or \textbf{umount} command instead. Note +\textbf{\texttt{fusermount -u}} or \textbf{\texttt{umount}} command instead. Note that these commands do not block until all data has been uploaded, so -if you use them instead of \code{umount.s3ql} then you should manually wait -for the \code{mount.s3ql} process to terminate before shutting down the +if you use them instead of \sphinxcode{umount.s3ql} then you should manually wait +for the \sphinxcode{mount.s3ql} process to terminate before shutting down the system. -The \textbf{umount.s3ql} command accepts the following options: +The \textbf{\texttt{umount.s3ql}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -1837,42 +1870,42 @@ \end{optionlist} \end{quote} -If, for some reason, the \code{umount.sql} command does not work, the file -system can also be unmounted with \code{fusermount -u -z}. Note that this +If, for some reason, the \sphinxcode{umount.sql} command does not work, the file +system can also be unmounted with \sphinxcode{fusermount -u -z}. Note that this command will return immediately and the file system may continue to upload data in the background for a while longer. \chapter{Checking for Errors} \label{fsck:checking-for-errors}\label{fsck::doc} -It is recommended to periodically run the \textbf{fsck.s3ql} and -\textbf{s3ql\_verify} commands (in this order) to ensure that the +It is recommended to periodically run the \textbf{\texttt{fsck.s3ql}} and +\textbf{\texttt{s3ql\_verify}} commands (in this order) to ensure that the file system is consistent, and that there has been no data corruption or data loss in the storage backend. -\textbf{fsck.s3ql} is intended to detect and correct problems with +\textbf{\texttt{fsck.s3ql}} is intended to detect and correct problems with the internal file system structure, caused by e.g. a file system crash or a bug in S3QL. It assumes that the storage backend can be fully trusted, i.e. if the backend reports that a specific storage object -exists, \textbf{fsck.s3ql} takes that as proof that the data is +exists, \textbf{\texttt{fsck.s3ql}} takes that as proof that the data is present and intact. -In contrast to that, the \textbf{s3ql\_verify} command is intended to +In contrast to that, the \textbf{\texttt{s3ql\_verify}} command is intended to check the consistency of the storage backend. It assumes that the internal file system data is correct, and verifies that all data can -actually be retrieved from the backend. Running \textbf{s3ql\_verify} -may therefore take much longer than running \textbf{fsck.s3ql}. +actually be retrieved from the backend. Running \textbf{\texttt{s3ql\_verify}} +may therefore take much longer than running \textbf{\texttt{fsck.s3ql}}. \section{Checking and repairing internal file system errors} \label{fsck:checking-and-repairing-internal-file-system-errors} -\textbf{fsck.s3ql} checks that the internal file system structure is +\textbf{\texttt{fsck.s3ql}} checks that the internal file system structure is consistent and attempts to correct any problems it finds. If an S3QL file system has not been unmounted correcly for any reason, you need -to run \textbf{fsck.s3ql} before you can mount the file system +to run \textbf{\texttt{fsck.s3ql}} before you can mount the file system again. -The \textbf{fsck.s3ql} command has the following syntax: +The \textbf{\texttt{fsck.s3ql}} command has the following syntax: \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{fsck.s3ql }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} @@ -1882,27 +1915,27 @@ \begin{quote} \begin{optionlist}{3cm} \item [-{-}log \textless{}target\textgreater{}] -Destination for log messages. Specify \code{none} for -standard output or \code{syslog} for the system logging +Destination for log messages. Specify \sphinxcode{none} for +standard output or \sphinxcode{syslog} for the system logging daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: -\code{\textasciitilde{}/.s3ql/fsck.log} +\sphinxcode{\textasciitilde{}/.s3ql/fsck.log} \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}backend-options \textless{}options\textgreater{}] @@ -1920,36 +1953,36 @@ \section{Detecting and handling backend data corruption} \label{fsck:s3ql-verify}\label{fsck:detecting-and-handling-backend-data-corruption} -The \textbf{s3ql\_verify} command verifies all data in the file -system. In contrast to \textbf{fsck.s3ql}, \textbf{s3ql\_verify} +The \textbf{\texttt{s3ql\_verify}} command verifies all data in the file +system. In contrast to \textbf{\texttt{fsck.s3ql}}, \textbf{\texttt{s3ql\_verify}} does not trust the object listing returned by the backend, but actually attempts to retrieve every object. By default, -\textbf{s3ql\_verify} will attempt to retrieve just the metadata for +\textbf{\texttt{s3ql\_verify}} will attempt to retrieve just the metadata for every object (for e.g. the S3-compatible or Google Storage backends -this corresponds to a \code{HEAD} request for each object), which is +this corresponds to a \sphinxcode{HEAD} request for each object), which is generally sufficient to determine if the object still exists. When -specifying the \code{-{-}data} option, \textbf{s3ql\_verify} will +specifying the \sphinxcode{-{-}data} option, \textbf{\texttt{s3ql\_verify}} will instead read every object entirely. To determine how much data will be -transmitted in total when using \code{-{-}data}, look at the \emph{After -compression} row in the {\hyperref[special:s3qlstat]{\emph{s3qlstat}}} output. +transmitted in total when using \sphinxcode{-{-}data}, look at the \emph{After +compression} row in the {\hyperref[special:s3qlstat]{\sphinxcrossref{\DUrole{std,std-ref}{s3qlstat}}}} output. -\textbf{s3ql\_verify} is not able to correct any data corruption that +\textbf{\texttt{s3ql\_verify}} is not able to correct any data corruption that it finds. Instead, a list of the corrupted and/or missing objects is written to a file and the decision about the proper course of action is left to the user. If you have administrative access to the backend server, you may want to investigate the cause of the corruption or check if the missing/corrupted objects can be restored from backups. If you believe that the missing/corrupted objects are indeed -lost irrevocably, you can use the {\hyperref[contrib:remove-objects]{\emph{remove\_objects.py}}} script (from -the \code{contrib} directory of the S3QL distribution) to explicitly +lost irrevocably, you can use the {\hyperref[contrib:remove\string-objects]{\sphinxcrossref{\DUrole{std,std-ref}{remove\_objects.py}}}} script (from +the \sphinxcode{contrib} directory of the S3QL distribution) to explicitly delete the objects from the storage backend. After that, you should -run \textbf{fsck.s3ql}. Since the (now explicitly deleted) objects +run \textbf{\texttt{fsck.s3ql}}. Since the (now explicitly deleted) objects should now no longer be included in the object index reported by the -backend, \textbf{fsck.s3ql} will identify the objects as missing, +backend, \textbf{\texttt{fsck.s3ql}} will identify the objects as missing, update the internal file system structures accordingly, and move the -affected files into the \code{lost+found} directory. +affected files into the \sphinxcode{lost+found} directory. -The \textbf{s3ql\_verify} command has the following syntax: +The \textbf{\texttt{s3ql\_verify}} command has the following syntax: \begin{Verbatim}[commandchars=\\\{\}] \PYG{l}{s3ql\PYGZus{}verify }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} @@ -1962,21 +1995,21 @@ Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] just print program version and exit \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. @@ -1997,14 +2030,14 @@ \chapter{Storing Authentication Information} -\label{authinfo:authinfo}\label{authinfo:storing-authentication-information}\label{authinfo::doc} +\label{authinfo:storing-authentication-information}\label{authinfo:authinfo}\label{authinfo::doc} Normally, S3QL reads username and password for the backend as well as an encryption passphrase for the file system from the terminal. Most -commands also accept an \code{-{-}authfile} parameter that can be +commands also accept an \sphinxcode{-{-}authfile} parameter that can be used to read this information from a file instead. -The authentication file consists of sections, led by a \code{{[}section{]}} -header and followed by \code{name: value} entries. The section headers +The authentication file consists of sections, led by a \sphinxcode{{[}section{]}} +header and followed by \sphinxcode{name: value} entries. The section headers themselves are not used by S3QL but have to be unique within the file. In each section, the following entries can be defined: @@ -2068,8 +2101,8 @@ S3QL comes with a few contributed programs that are not part of the core distribution (and are therefore not installed automatically by default), but which may nevertheless be useful. These programs are in -the \code{contrib} directory of the source distribution or in -\code{/usr/share/doc/s3ql/contrib} if you installed S3QL from a package. +the \sphinxcode{contrib} directory of the source distribution or in +\sphinxcode{/usr/share/doc/s3ql/contrib} if you installed S3QL from a package. \section{benchmark.py} @@ -2089,15 +2122,15 @@ \section{pcp.py} -\label{contrib:pcp}\label{contrib:pcp-py} -\code{pcp.py} is a wrapper program that starts several rsync processes to +\label{contrib:pcp-py}\label{contrib:pcp} +\sphinxcode{pcp.py} is a wrapper program that starts several rsync processes to copy directory trees in parallel. This is important because transferring files in parallel significantly enhances performance when -copying data from an S3QL file system (see {\hyperref[tips:copy-performance]{\emph{Improving copy performance}}} for +copying data from an S3QL file system (see {\hyperref[tips:copy\string-performance]{\sphinxcrossref{\DUrole{std,std-ref}{Improving copy performance}}}} for details). -To recursively copy the directory \code{/mnt/home-backup} into -\code{/home/joe} using 8 parallel processes and preserving permissions, +To recursively copy the directory \sphinxcode{/mnt/home-backup} into +\sphinxcode{/home/joe} using 8 parallel processes and preserving permissions, you would execute \begin{Verbatim}[commandchars=\\\{\}] @@ -2110,19 +2143,19 @@ This is an example script that demonstrates how to set up a simple but powerful backup solution using S3QL and \href{http://samba.org/rsync}{rsync}. -The \code{s3ql\_backup.sh} script automates the following steps: +The \sphinxcode{s3ql\_backup.sh} script automates the following steps: \begin{enumerate} \item {} Mount the file system \item {} -Replicate the previous backup with {\hyperref[special:s3qlcp]{\emph{s3qlcp}}} +Replicate the previous backup with {\hyperref[special:s3qlcp]{\sphinxcrossref{\DUrole{std,std-ref}{s3qlcp}}}} \item {} Update the new copy with the data from the backup source using rsync \item {} -Make the new backup immutable with {\hyperref[special:s3qllock]{\emph{s3qllock}}} +Make the new backup immutable with {\hyperref[special:s3qllock]{\sphinxcrossref{\DUrole{std,std-ref}{s3qllock}}}} \item {} Delete old backups that are no longer needed @@ -2133,17 +2166,17 @@ \end{enumerate} The backups are stored in directories of the form -\code{YYYY-MM-DD\_HH:mm:SS} and the {\hyperref[contrib:expire-backups-py]{expire\_backups.py}} command is used to +\sphinxcode{YYYY-MM-DD\_HH:mm:SS} and the {\hyperref[contrib:expire\string-backups\string-py]{\sphinxcrossref{expire\_backups.py}}} command is used to delete old backups. \section{expire\_backups.py} \label{contrib:expire-backups-py} -\textbf{expire\_backups.py} is a program to intelligently remove old +\textbf{\texttt{expire\_backups.py}} is a program to intelligently remove old backups that are no longer needed. To define what backups you want to keep for how long, you define a -number of \emph{age ranges}. \textbf{expire\_backups} ensures that you +number of \emph{age ranges}. \textbf{\texttt{expire\_backups}} ensures that you will have at least one backup in each age range at all times. It will keep exactly as many backups as are required for that and delete any backups that become redundant. @@ -2152,8 +2185,8 @@ of backup cycles. Every time you create a new backup, the existing backups age by one cycle. -Example: when \textbf{expire\_backups} is called with the age range -definition \code{1 3 7 14 31}, it will guarantee that you always have the +Example: when \textbf{\texttt{expire\_backups}} is called with the age range +definition \sphinxcode{1 3 7 14 31}, it will guarantee that you always have the following backups available: \begin{enumerate} \item {} @@ -2185,9 +2218,9 @@ cycles avoids these sort of problems. \end{notice} -\textbf{expire\_backups} usage is simple. It requires backups to be -stored in directories of the form \code{year-month-day\_hour:minute:seconds} -(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current +\textbf{\texttt{expire\_backups}} usage is simple. It requires backups to be +stored in directories of the form \sphinxcode{year-month-day\_hour:minute:seconds} +(\sphinxcode{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current directory. So for the above backup strategy, the correct invocation would be: @@ -2196,30 +2229,30 @@ \end{Verbatim} When storing your backups on an S3QL file system, you probably want to -specify the \code{-{-}use-s3qlrm} option as well. This tells -\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to +specify the \sphinxcode{-{-}use-s3qlrm} option as well. This tells +\textbf{\texttt{expire\_backups}} to use the {\hyperref[special:s3qlrm]{\sphinxcrossref{\DUrole{std,std-ref}{s3qlrm}}}} command to delete directories. -\textbf{expire\_backups} uses a ``state file'' to keep track which +\textbf{\texttt{expire\_backups}} uses a ``state file'' to keep track which backups are how many cycles old (since this cannot be inferred from the dates contained in the directory names). The standard name for -this state file is \code{.expire\_backups.dat}. If this file gets -damaged or deleted, \textbf{expire\_backups} no longer knows the ages +this state file is \sphinxcode{.expire\_backups.dat}. If this file gets +damaged or deleted, \textbf{\texttt{expire\_backups}} no longer knows the ages of the backups and refuses to work. In this case you can use the -\code{-{-}reconstruct-state} option to try to reconstruct the state +\sphinxcode{-{-}reconstruct-state} option to try to reconstruct the state from the backup dates. However, the accuracy of this reconstruction depends strongly on how rigorous you have been with making backups (it is only completely correct if the time between subsequent backups has always been exactly the same), so it's generally a good idea not to tamper with the state file. -For a full list of available options, run \textbf{expire\_backups.py ---help}. +For a full list of available options, run \textbf{\texttt{expire\_backups.py +-{-}help}}. \section{remove\_objects.py} -\label{contrib:remove-objects}\label{contrib:remove-objects-py} -\textbf{remove\_objects.py} is a program to remove a list of objects +\label{contrib:remove-objects-py}\label{contrib:remove-objects} +\textbf{\texttt{remove\_objects.py}} is a program to remove a list of objects from a storage backend. Since it acts on the backend-level, the backend need not contain an S3QL file system. @@ -2228,7 +2261,7 @@ \label{tips:tips-tricks}\label{tips::doc} \section{SSH Backend} -\label{tips:ssh-tipp}\label{tips:ssh-backend} +\label{tips:ssh-backend}\label{tips:ssh-tipp} By combining S3QL's local backend with \href{http://fuse.sourceforge.net/sshfs.html}{sshfs}, it is possible to store an S3QL file system on arbitrary SSH servers: first mount the remote target directory into the local filesystem, @@ -2255,14 +2288,14 @@ If you decide to do so, you should make sure to \begin{itemize} \item {} -Use {\hyperref[special:s3qllock]{\emph{s3qllock}}} to ensure that backups are immutable +Use {\hyperref[special:s3qllock]{\sphinxcrossref{\DUrole{std,std-ref}{s3qllock}}}} to ensure that backups are immutable after they have been made. \item {} -Call {\hyperref[special:s3qlctrl]{\emph{s3qlctrl upload-meta}}} right after a every +Call {\hyperref[special:s3qlctrl]{\sphinxcrossref{\DUrole{std,std-ref}{s3qlctrl upload-meta}}}} right after a every backup to make sure that the newest metadata is stored safely (if you do backups often enough, this may also allow you to set the -\code{-{-}metadata-upload-interval} option of \textbf{mount.s3ql} +\sphinxcode{-{-}metadata-upload-interval} option of \textbf{\texttt{mount.s3ql}} to zero). \end{itemize} @@ -2288,7 +2321,7 @@ Theoretically, this problem is easy to solve: you just have to copy several files at the same time. In practice, however, almost all unix -utilities (\code{cp}, \code{rsync}, \code{tar} and friends) insist on copying +utilities (\sphinxcode{cp}, \sphinxcode{rsync}, \sphinxcode{tar} and friends) insist on copying data one file at a time. This makes a lot of sense when copying data on the local hard disk, but in case of S3QL this is really unfortunate. @@ -2299,7 +2332,7 @@ For example, the following script will start 3 rsync instances. The first instance handles all filenames starting with a-f, the second the -filenames from g-l and the third covers the rest. The \code{+ */} rule +filenames from g-l and the third covers the rest. The \sphinxcode{+ */} rule ensures that every instance looks into all directories. \begin{Verbatim}[commandchars=\\\{\}] @@ -2319,14 +2352,14 @@ However, starting about 10 processes seems to be a good compromise that increases performance dramatically in almost all situations. -S3QL comes with a script named \code{pcp.py} in the \code{contrib} directory +S3QL comes with a script named \sphinxcode{pcp.py} in the \sphinxcode{contrib} directory that can be used to transfer files in parallel without having to write -an explicit script first. See the description of {\hyperref[contrib:pcp]{\emph{pcp.py}}} for +an explicit script first. See the description of {\hyperref[contrib:pcp]{\sphinxcrossref{\DUrole{std,std-ref}{pcp.py}}}} for details. \chapter{Known Issues} -\label{issues:known-issues}\label{issues::doc}\begin{itemize} +\label{issues::doc}\label{issues:known-issues}\begin{itemize} \item {} S3QL de-duplicates data blocks based solely only on SHA256 checksums, without doing a byte-by-byte comparison of the blocks. @@ -2353,7 +2386,7 @@ the write protect bit set, the owner of the directory can delete any files and (empty) subdirectories inside it. This is a bug in the FUSE kernel module -(cf. \href{https://github.com/libfuse/libfuse/issues/23}{https://github.com/libfuse/libfuse/issues/23}) and needs to be +(cf. \url{https://github.com/libfuse/libfuse/issues/23}) and needs to be fixed in the kernel. Unfortunately it does not look as if this is going to be fixed anytime soon (as of 2016/2/28). @@ -2367,7 +2400,7 @@ write caching) which will hopefully be addressed in some future FUSE version. -Most applications, including e.g. GNU \code{cp} and \code{rsync}, use +Most applications, including e.g. GNU \sphinxcode{cp} and \sphinxcode{rsync}, use reasonably large buffers and are therefore not affected by this problem and perform very efficient on S3QL file systems. @@ -2377,34 +2410,34 @@ it might be worth to ask the program's authors for help. \item {} -S3QL always updates file and directory access times as if the \code{relatime} +S3QL always updates file and directory access times as if the \sphinxcode{relatime} mount option has been specified: the access time (``atime'') is only updated if it is currently earlier than either the status change time (``ctime'') or modification time (``mtime''). \item {} -S3QL directories always have an \code{st\_nlink} value of 1. This may confuse -programs that rely on directories having \code{st\_nlink} values of \emph{(2 + +S3QL directories always have an \sphinxcode{st\_nlink} value of 1. This may confuse +programs that rely on directories having \sphinxcode{st\_nlink} values of \emph{(2 + number of sub directories)}. Note that this is not a bug in S3QL. Including sub directories in -the \code{st\_nlink} value is a Unix convention, but by no means a +the \sphinxcode{st\_nlink} value is a Unix convention, but by no means a requirement. If an application blindly relies on this convention being followed, then this is a bug in the application. A prominent example are early versions of GNU find, which required -the \code{-{-}noleaf} option to work correctly on S3QL file systems. This +the \sphinxcode{-{-}noleaf} option to work correctly on S3QL file systems. This bug has already been fixed in recent find versions. \item {} -The \code{umount} and \code{fusermount -u} commands will \emph{not} block until all +The \sphinxcode{umount} and \sphinxcode{fusermount -u} commands will \emph{not} block until all data has been uploaded to the backend. (this is a FUSE limitation that will hopefully be removed in the future, see \href{https://bitbucket.org/nikratio/s3ql/issue/1/blocking-fusermount-and-umount}{issue \#1}). If you use either command to unmount an S3QL file system, you have to -take care to explicitly wait for the \code{mount.s3ql} process to +take care to explicitly wait for the \sphinxcode{mount.s3ql} process to terminate before you shut down or restart the system. Therefore it is generally not a good idea to mount an S3QL file system in -\code{/etc/fstab} (you should use a dedicated init script instead). +\sphinxcode{/etc/fstab} (you should use a dedicated init script instead). \item {} S3QL relies on the backends not to run out of space. This is a given @@ -2433,13 +2466,13 @@ \chapter{Manpages} -\label{man/index:manpages}\label{man/index::doc} +\label{man/index::doc}\label{man/index:manpages} The man pages are installed with S3QL on your system and can be viewed -with the \textbf{man} command. For reference, they are also included +with the \textbf{\texttt{man}} command. For reference, they are also included here in the User's Guide. -\section{The \textbf{mkfs.s3ql} command} +\section{The \textbf{\texttt{mkfs.s3ql}} command} \label{man/mkfs:the-command-command}\label{man/mkfs::doc} \subsection{Synopsis} @@ -2451,37 +2484,37 @@ \subsection{Description} \label{man/mkfs:description} -The \textbf{mkfs.s3ql} command creates a new file system in the location +The \textbf{\texttt{mkfs.s3ql}} command creates a new file system in the location specified by \emph{storage url}. The storage url depends on the backend that is used. The S3QL User's Guide should be consulted for a description of the available backends. -Unless you have specified the \code{-{-}plain} option, \code{mkfs.s3ql} will ask +Unless you have specified the \sphinxcode{-{-}plain} option, \sphinxcode{mkfs.s3ql} will ask you to enter an encryption password. This password will \emph{not} be read -from an authentication file specified with the \code{-{-}authfile} +from an authentication file specified with the \sphinxcode{-{-}authfile} option to prevent accidental creation of an encrypted file system. \subsection{Options} \label{man/mkfs:options} -The \textbf{mkfs.s3ql} command accepts the following options. +The \textbf{\texttt{mkfs.s3ql}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}backend-options \textless{}options\textgreater{}] @@ -2505,7 +2538,7 @@ \subsection{Exit Codes} \label{man/mkfs:exit-codes} -\textbf{mkfs.s3ql} may terminate with the following exit codes: +\textbf{\texttt{mkfs.s3ql}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -2549,14 +2582,14 @@ \subsection{See Also} \label{man/mkfs:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3qladm} command} +\section{The \textbf{\texttt{s3qladm}} command} \label{man/adm:the-command-command}\label{man/adm::doc} \subsection{Synopsis} @@ -2565,15 +2598,15 @@ \PYG{l}{s3qladm }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}storage url\PYGZgt{}} \end{Verbatim} -where \code{action} may be either of \textbf{passphrase}, -\textbf{upgrade}, \textbf{delete} or \textbf{download-metadata}. +where \sphinxcode{action} may be either of \textbf{\texttt{passphrase}}, +\textbf{\texttt{upgrade}}, \textbf{\texttt{delete}} or \textbf{\texttt{download-metadata}}. \subsection{Description} \label{man/adm:description} -The \textbf{s3qladm} command performs various operations on \emph{unmounted} S3QL +The \textbf{\texttt{s3qladm}} command performs various operations on \emph{unmounted} S3QL file systems. The file system \emph{must not be mounted} when using -\textbf{s3qladm} or things will go wrong badly. +\textbf{\texttt{s3qladm}} or things will go wrong badly. The storage url depends on the backend that is used. The S3QL User's Guide should be consulted for a description of the available backends. @@ -2581,42 +2614,42 @@ \subsection{Options} \label{man/adm:options} -The \textbf{s3qladm} command accepts the following options. +The \textbf{\texttt{s3qladm}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}log \textless{}target\textgreater{}] -Destination for log messages. Specify \code{none} for -standard output or \code{syslog} for the system logging +Destination for log messages. Specify \sphinxcode{none} for +standard output or \sphinxcode{syslog} for the system logging daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: -\code{None} +\sphinxcode{None} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}version] just print program version and exit \end{optionlist} \end{quote} -Hint: run \code{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments +Hint: run \sphinxcode{s3qladm \textless{}action\textgreater{} -{-}help} to get help on the additional arguments that the different actions take. @@ -2641,7 +2674,7 @@ \subsection{Exit Codes} \label{man/adm:exit-codes} -\textbf{s3qladm} may terminate with the following exit codes: +\textbf{\texttt{s3qladm}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -2694,14 +2727,14 @@ \subsection{See Also} \label{man/adm:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{mount.s3ql} command} +\section{The \textbf{\texttt{mount.s3ql}} command} \label{man/mount:the-command-command}\label{man/mount::doc} \subsection{Synopsis} @@ -2713,7 +2746,7 @@ \subsection{Description} \label{man/mount:description} -The \textbf{mount.s3ql} command mounts the S3QL file system stored in \emph{storage +The \textbf{\texttt{mount.s3ql}} command mounts the S3QL file system stored in \emph{storage url} in the directory \emph{mount point}. The storage url depends on the backend that is used. The S3QL User's Guide should be consulted for a description of the available backends. @@ -2721,31 +2754,31 @@ \subsection{Options} \label{man/mount:options} -The \textbf{mount.s3ql} command accepts the following options. +The \textbf{\texttt{mount.s3ql}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}log \textless{}target\textgreater{}] -Destination for log messages. Specify \code{none} for -standard output or \code{syslog} for the system logging +Destination for log messages. Specify \sphinxcode{none} for +standard output or \sphinxcode{syslog} for the system logging daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: -\code{\textasciitilde{}/.s3ql/mount.log} +\sphinxcode{\textasciitilde{}/.s3ql/mount.log} \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}backend-options \textless{}options\textgreater{}] @@ -2760,18 +2793,18 @@ autodetect). Each cache entry requires one file descriptor, so if you increase this number you have to make sure that your process file descriptor limit (as -set with \code{ulimit -n}) is high enough (at least the +set with \sphinxcode{ulimit -n}) is high enough (at least the number of cache entries + 100). \item [-{-}allow-other] -Normally, only the user who called \code{mount.s3ql} can +Normally, only the user who called \sphinxcode{mount.s3ql} can access the mount point. This user then also has full access to it, independent of individual file -permissions. If the \code{-{-}allow-other} option is +permissions. If the \sphinxcode{-{-}allow-other} option is specified, other users can access the mount point as well and individual file permissions are taken into account for all users. \item [-{-}allow-root] -Like \code{-{-}allow-other}, but restrict access to the +Like \sphinxcode{-{-}allow-other}, but restrict access to the mounting user and the root user. \item [-{-}fg] Do not daemonize, stay in foreground @@ -2781,9 +2814,9 @@ \item [-{-}compress \textless{}algorithm-lvl\textgreater{}] Compression algorithm and compression level to use when storing new data. \emph{algorithm} may be any of -\code{lzma}, \code{bzip2}, \code{zlib}, or none. \emph{lvl} may be any +\sphinxcode{lzma}, \sphinxcode{bzip2}, \sphinxcode{zlib}, or none. \emph{lvl} may be any integer from 0 (fastest) to 9 (slowest). Default: -\code{lzma-6} +\sphinxcode{lzma-6} \item [-{-}metadata-upload-interval \textless{}seconds\textgreater{}] Interval in seconds between complete metadata uploads. Set to 0 to disable. Default: 24h. @@ -2799,7 +2832,7 @@ \subsection{Exit Codes} \label{man/mount:exit-codes} -\textbf{mount.s3ql} may terminate with the following exit codes: +\textbf{\texttt{mount.s3ql}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -2857,7 +2890,7 @@ Unsupported file system revision (too new). \item[{34}] \leavevmode -Insufficient free nodes, need to run \textbf{fsck.s3ql}. +Insufficient free nodes, need to run \textbf{\texttt{fsck.s3ql}}. \item[{35}] \leavevmode Attempted to mount read-only, this is not supported. @@ -2879,14 +2912,14 @@ \subsection{See Also} \label{man/mount:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3qlstat} command} +\section{The \textbf{\texttt{s3qlstat}} command} \label{man/stat:the-command-command}\label{man/stat::doc} \subsection{Synopsis} @@ -2898,28 +2931,28 @@ \subsection{Description} \label{man/stat:description} -The \textbf{s3qlstat} command prints statistics about the S3QL file system mounted -at \code{mountpoint}. +The \textbf{\texttt{s3qlstat}} command prints statistics about the S3QL file system mounted +at \sphinxcode{mountpoint}. -\textbf{s3qlstat} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or -\code{-{-}allow-root}) the root user. +\textbf{\texttt{s3qlstat}} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or +\sphinxcode{-{-}allow-root}) the root user. \subsection{Options} \label{man/stat:options} -The \textbf{s3qlstat} command accepts the following options: +The \textbf{\texttt{s3qlstat}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -2932,7 +2965,7 @@ \subsection{Exit Codes} \label{man/stat:exit-codes} -\textbf{s3qlstat} may terminate with the following exit codes: +\textbf{\texttt{s3qlstat}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -2949,14 +2982,14 @@ \subsection{See Also} \label{man/stat:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3qlctrl} command} +\section{The \textbf{\texttt{s3qlctrl}} command} \label{man/ctrl:the-command-command}\label{man/ctrl::doc} \subsection{Synopsis} @@ -2965,19 +2998,19 @@ \PYG{l}{s3qlctrl }\PYG{g+ge}{[options]}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}action\PYGZgt{}}\PYG{l}{ }\PYG{n+nv}{\PYGZlt{}mountpoint\PYGZgt{}}\PYG{l}{ ...} \end{Verbatim} -where \code{action} may be either of \textbf{flushcache}, -\textbf{upload-meta}, \textbf{cachesize} or -\textbf{log-metadata}. +where \sphinxcode{action} may be either of \textbf{\texttt{flushcache}}, +\textbf{\texttt{upload-meta}}, \textbf{\texttt{cachesize}} or +\textbf{\texttt{log-metadata}}. \subsection{Description} \label{man/ctrl:description} -The \textbf{s3qlctrl} command performs various actions on the S3QL file system mounted -in \code{mountpoint}. +The \textbf{\texttt{s3qlctrl}} command performs various actions on the S3QL file system mounted +in \sphinxcode{mountpoint}. -\textbf{s3qlctrl} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or -\code{-{-}allow-root}) the root user. +\textbf{\texttt{s3qlctrl}} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or +\sphinxcode{-{-}allow-root}) the root user. The following actions may be specified: \begin{description} @@ -2999,14 +3032,14 @@ \item[{log}] \leavevmode Change the amount of information that is logged into -\code{\textasciitilde{}/.s3ql/mount.log} file. The complete syntax is: +\sphinxcode{\textasciitilde{}/.s3ql/mount.log} file. The complete syntax is: \begin{Verbatim}[commandchars=\\\{\}] s3qlctrl [options] log \PYGZlt{}mountpoint\PYGZgt{} \PYGZlt{}level\PYGZgt{} [\PYGZlt{}module\PYGZgt{} [\PYGZlt{}module\PYGZgt{} ...]] \end{Verbatim} -here \code{level} is the desired new log level and may be either of -\emph{debug}, \emph{info} or \emph{warn}. One or more \code{module} may only be +here \sphinxcode{level} is the desired new log level and may be either of +\emph{debug}, \emph{info} or \emph{warn}. One or more \sphinxcode{module} may only be specified with the \emph{debug} level and allow to restrict the debug output to just the listed modules. @@ -3015,7 +3048,7 @@ \subsection{Options} \label{man/ctrl:options} -The \textbf{s3qlctrl} command also accepts the following options, no matter +The \textbf{\texttt{s3qlctrl}} command also accepts the following options, no matter what specific action is being invoked: \begin{quote} \begin{optionlist}{3cm} @@ -3023,11 +3056,11 @@ Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -3035,13 +3068,13 @@ \end{optionlist} \end{quote} -Hint: run \code{s3qlctrl \textless{}action\textgreater{} -{-}help} to get help on the additional arguments +Hint: run \sphinxcode{s3qlctrl \textless{}action\textgreater{} -{-}help} to get help on the additional arguments that the different actions take. \subsection{Exit Codes} \label{man/ctrl:exit-codes} -\textbf{s3qlctrl} may terminate with the following exit codes: +\textbf{\texttt{s3qlctrl}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3058,14 +3091,14 @@ \subsection{See Also} \label{man/ctrl:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3qlcp} command} +\section{The \textbf{\texttt{s3qlcp}} command} \label{man/cp:the-command-command}\label{man/cp::doc} \subsection{Synopsis} @@ -3077,36 +3110,36 @@ \subsection{Description} \label{man/cp:description} -The \textbf{s3qlcp} command duplicates the directory tree \code{source-dir} -into \code{dest-dir} without physically copying the file contents. +The \textbf{\texttt{s3qlcp}} command duplicates the directory tree \sphinxcode{source-dir} +into \sphinxcode{dest-dir} without physically copying the file contents. Both source and destination must lie inside the same S3QL file system. The replication will not take any additional space. Only if one of directories is modified later on, the modified data will take additional storage space. -\code{s3qlcp} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or \code{-{-}allow-root}) +\sphinxcode{s3qlcp} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or \sphinxcode{-{-}allow-root}) the root user. Note that: \begin{itemize} \item {} After the replication, both source and target directory will still -be completely ordinary directories. You can regard \code{\textless{}src\textgreater{}} as a -snapshot of \code{\textless{}target\textgreater{}} or vice versa. However, the most common -usage of \code{s3qlcp} is to regularly duplicate the same source -directory, say \code{documents}, to different target directories. For a +be completely ordinary directories. You can regard \sphinxcode{\textless{}src\textgreater{}} as a +snapshot of \sphinxcode{\textless{}target\textgreater{}} or vice versa. However, the most common +usage of \sphinxcode{s3qlcp} is to regularly duplicate the same source +directory, say \sphinxcode{documents}, to different target directories. For a e.g. monthly replication, the target directories would typically be -named something like \code{documents\_January} for the replication in -January, \code{documents\_February} for the replication in February etc. +named something like \sphinxcode{documents\_January} for the replication in +January, \sphinxcode{documents\_February} for the replication in February etc. In this case it is clear that the target directories should be regarded as snapshots of the source directory. \item {} Exactly the same effect could be achieved by an ordinary copy -program like \code{cp -a}. However, this procedure would be orders of -magnitude slower, because \code{cp} would have to read every file +program like \sphinxcode{cp -a}. However, this procedure would be orders of +magnitude slower, because \sphinxcode{cp} would have to read every file completely (so that S3QL had to fetch all the data over the network from the backend) before writing them into the destination folder. @@ -3141,18 +3174,18 @@ \subsection{Options} \label{man/cp:options} -The \textbf{s3qlcp} command accepts the following options: +The \textbf{\texttt{s3qlcp}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -3163,7 +3196,7 @@ \subsection{Exit Codes} \label{man/cp:exit-codes} -\textbf{s3qlcp} may terminate with the following exit codes: +\textbf{\texttt{s3qlcp}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3180,14 +3213,14 @@ \subsection{See Also} \label{man/cp:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3qlrm} command} +\section{The \textbf{\texttt{s3qlrm}} command} \label{man/rm:the-command-command}\label{man/rm::doc} \subsection{Synopsis} @@ -3199,34 +3232,34 @@ \subsection{Description} \label{man/rm:description} -The \textbf{s3qlrm} command recursively deletes files and directories on an -S3QL file system. Although \textbf{s3qlrm} is faster than using e.g. -\textbf{rm -r{}`}, the main reason for its existence is that it allows +The \textbf{\texttt{s3qlrm}} command recursively deletes files and directories on an +S3QL file system. Although \textbf{\texttt{s3qlrm}} is faster than using e.g. +\textbf{\texttt{rm -r{}`}}, the main reason for its existence is that it allows you to delete immutable trees (which can be created with -\textbf{s3qllock}) as well. +\textbf{\texttt{s3qllock}}) as well. Be warned that there is no additional confirmation. The directory will be removed entirely and immediately. -\textbf{s3qlrm} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or -\code{-{-}allow-root}) the root user. +\textbf{\texttt{s3qlrm}} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or +\sphinxcode{-{-}allow-root}) the root user. \subsection{Options} \label{man/rm:options} -The \textbf{s3qlrm} command accepts the following options: +The \textbf{\texttt{s3qlrm}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -3237,7 +3270,7 @@ \subsection{Exit Codes} \label{man/rm:exit-codes} -\textbf{s3qlrm} may terminate with the following exit codes: +\textbf{\texttt{s3qlrm}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3254,14 +3287,14 @@ \subsection{See Also} \label{man/rm:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3qllock} command} +\section{The \textbf{\texttt{s3qllock}} command} \label{man/lock:the-command-command}\label{man/lock::doc} \subsection{Synopsis} @@ -3273,15 +3306,15 @@ \subsection{Description} \label{man/lock:description} -The \textbf{s3qllock} command makes a directory tree in an S3QL file +The \textbf{\texttt{s3qllock}} command makes a directory tree in an S3QL file system immutable. Immutable trees can no longer be changed in any way whatsoever. You can not add new files or directories and you can not change or delete existing files and directories. The only way to get -rid of an immutable tree is to use the \textbf{s3qlrm} command. +rid of an immutable tree is to use the \textbf{\texttt{s3qlrm}} command. -\textbf{s3qllock} can only be called by the user that mounted the file system -and (if the file system was mounted with \code{-{-}allow-other} or -\code{-{-}allow-root}) the root user. +\textbf{\texttt{s3qllock}} can only be called by the user that mounted the file system +and (if the file system was mounted with \sphinxcode{-{-}allow-other} or +\sphinxcode{-{-}allow-root}) the root user. \subsection{Rationale} @@ -3306,7 +3339,7 @@ excluded, being able to change a backup after it has been made is generally not a good idea. A common S3QL use case is to keep the file system mounted at all times and periodically create backups with -\textbf{rsync -a}. This allows every user to recover her files from a +\textbf{\texttt{rsync -a}}. This allows every user to recover her files from a backup without having to call the system administrator. However, this also allows every user to accidentally change or delete files \emph{in} one of the old backups. @@ -3319,18 +3352,18 @@ \subsection{Options} \label{man/lock:options} -The \textbf{s3qllock} command accepts the following options: +The \textbf{\texttt{s3qllock}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -3341,7 +3374,7 @@ \subsection{Exit Codes} \label{man/lock:exit-codes} -\textbf{s3qllock} may terminate with the following exit codes: +\textbf{\texttt{s3qllock}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3358,14 +3391,14 @@ \subsection{See Also} \label{man/lock:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{umount.s3ql} command} +\section{The \textbf{\texttt{umount.s3ql}} command} \label{man/umount:the-command-command}\label{man/umount::doc} \subsection{Synopsis} @@ -3377,34 +3410,34 @@ \subsection{Description} \label{man/umount:description} -The \textbf{umount.s3ql} command unmounts the S3QL file system mounted in the +The \textbf{\texttt{umount.s3ql}} command unmounts the S3QL file system mounted in the directory \emph{mount point} and blocks until all data has been uploaded to the storage backend. -Only the user who mounted the file system with \textbf{mount.s3ql} -is able to unmount it with \textbf{umount.s3ql}. If you are root and want to +Only the user who mounted the file system with \textbf{\texttt{mount.s3ql}} +is able to unmount it with \textbf{\texttt{umount.s3ql}}. If you are root and want to unmount an S3QL file system mounted by an ordinary user, you have to -use the \textbf{fusermount -u} or \textbf{umount} command instead. +use the \textbf{\texttt{fusermount -u}} or \textbf{\texttt{umount}} command instead. Note that these commands do not block until all data has been -uploaded, so if you use them instead of \textbf{umount.s3ql} then -you should manually wait for the \textbf{mount.s3ql} process to +uploaded, so if you use them instead of \textbf{\texttt{umount.s3ql}} then +you should manually wait for the \textbf{\texttt{mount.s3ql}} process to terminate before shutting down the system. \subsection{Options} \label{man/umount:options} -The \textbf{umount.s3ql} command accepts the following options. +The \textbf{\texttt{umount.s3ql}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -3420,7 +3453,7 @@ \subsection{Exit Codes} \label{man/umount:exit-codes} -\textbf{umount.s3ql} may terminate with the following exit codes: +\textbf{\texttt{umount.s3ql}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3437,14 +3470,14 @@ \subsection{See Also} \label{man/umount:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{fsck.s3ql} command} +\section{The \textbf{\texttt{fsck.s3ql}} command} \label{man/fsck:the-command-command}\label{man/fsck::doc} \subsection{Synopsis} @@ -3456,7 +3489,7 @@ \subsection{Description} \label{man/fsck:description} -The \textbf{fsck.s3ql} command checks the file system in the location specified +The \textbf{\texttt{fsck.s3ql}} command checks the file system in the location specified by \emph{storage url} for errors and attempts to repair any problems. The storage url depends on the backend that is used. The S3QL User's Guide should be consulted for a description of the available backends. @@ -3464,31 +3497,31 @@ \subsection{Options} \label{man/fsck:options} -The \textbf{fsck.s3ql} command accepts the following options. +The \textbf{\texttt{fsck.s3ql}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}log \textless{}target\textgreater{}] -Destination for log messages. Specify \code{none} for -standard output or \code{syslog} for the system logging +Destination for log messages. Specify \sphinxcode{none} for +standard output or \sphinxcode{syslog} for the system logging daemon. Anything else will be interpreted as a file name. Log files will be rotated when they reach 1 MiB, and at most 5 old log files will be kept. Default: -\code{\textasciitilde{}/.s3ql/fsck.log} +\sphinxcode{\textasciitilde{}/.s3ql/fsck.log} \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}backend-options \textless{}options\textgreater{}] @@ -3506,7 +3539,7 @@ \subsection{Exit Codes} \label{man/fsck:exit-codes} -If \textbf{fsck.s3ql} found any file system errors (no matter if they were +If \textbf{\texttt{fsck.s3ql}} found any file system errors (no matter if they were corrected or not), the exit code will be 128 plus one of the codes listed below. If no errors were found, the following exit codes are used as-is: @@ -3588,15 +3621,15 @@ \subsection{See Also} \label{man/fsck:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3ql\_oauth\_client} command} -\label{man/oauth_client:the-command-command}\label{man/oauth_client:oauth-client}\label{man/oauth_client::doc} +\section{The \textbf{\texttt{s3ql\_oauth\_client}} command} +\label{man/oauth_client:the-command-command}\label{man/oauth_client::doc}\label{man/oauth_client:oauth-client} \subsection{Synopsis} \label{man/oauth_client:synopsis} @@ -3607,32 +3640,32 @@ \subsection{Description} \label{man/oauth_client:description} -The \textbf{s3ql\_oauth\_client} command may be used to obtain OAuth2 authentication +The \textbf{\texttt{s3ql\_oauth\_client}} command may be used to obtain OAuth2 authentication tokens for use with Google Storage. It requests ``user code'' from Google which has to be pasted into the browser to complete the authentication process interactively. Once authentication in the -browser has been completed, \textbf{s3ql\_oauth\_client} displays the OAuth2 refresh +browser has been completed, \textbf{\texttt{s3ql\_oauth\_client}} displays the OAuth2 refresh token. -When combined with the special username \code{oauth2}, the refresh token +When combined with the special username \sphinxcode{oauth2}, the refresh token can be used as a backend passphrase when using the Google Storage S3QL backend. \subsection{Options} \label{man/oauth_client:options} -The \textbf{s3ql\_oauth\_client} command accepts the following options: +The \textbf{\texttt{s3ql\_oauth\_client}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] @@ -3643,7 +3676,7 @@ \subsection{Exit Codes} \label{man/oauth_client:exit-codes} -\textbf{s3ql\_oauth\_client} may terminate with the following exit codes: +\textbf{\texttt{s3ql\_oauth\_client}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3660,14 +3693,14 @@ \subsection{See Also} \label{man/oauth_client:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{s3ql\_verify} command} +\section{The \textbf{\texttt{s3ql\_verify}} command} \label{man/verify:the-command-command}\label{man/verify::doc} \subsection{Synopsis} @@ -3679,40 +3712,40 @@ \subsection{Description} \label{man/verify:description} -The \textbf{s3ql\_verify} command verifies all data in the file system. In -contrast to \textbf{fsck.s3ql}, \textbf{s3ql\_verify} does not trust the object +The \textbf{\texttt{s3ql\_verify}} command verifies all data in the file system. In +contrast to \textbf{\texttt{fsck.s3ql}}, \textbf{\texttt{s3ql\_verify}} does not trust the object listing returned by the backend, but actually attempts to retrieve every object. It therefore takes a lot longer. -The format of \code{\textless{}storage url\textgreater{}} depends on the backend that is +The format of \sphinxcode{\textless{}storage url\textgreater{}} depends on the backend that is used. The S3QL User's Guide should be consulted for a description of the available backends. \subsection{Options} \label{man/verify:options} -The \textbf{s3ql\_verify} command accepts the following options. +The \textbf{\texttt{s3ql\_verify}} command accepts the following options. \begin{quote} \begin{optionlist}{3cm} \item [-{-}debug-modules \textless{}modules\textgreater{}] Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}quiet] be really quiet \item [-{-}version] just print program version and exit \item [-{-}cachedir \textless{}path\textgreater{}] Store cached data in this directory (default: -\code{\textasciitilde{}/.s3ql)} +\sphinxcode{\textasciitilde{}/.s3ql)} \item [-{-}authfile \textless{}path\textgreater{}] Read authentication credentials from this file -(default: \code{\textasciitilde{}/.s3ql/authinfo2)} +(default: \sphinxcode{\textasciitilde{}/.s3ql/authinfo2)} \item [-{-}backend-options \textless{}options\textgreater{}] Backend specific options (separate by commas). See backend documentation for available options. @@ -3734,7 +3767,7 @@ \subsection{Exit Codes} \label{man/verify:exit-codes} -\textbf{s3ql\_verify} may terminate with the following exit codes: +\textbf{\texttt{s3ql\_verify}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3797,14 +3830,14 @@ \subsection{See Also} \label{man/verify:see-also} -The S3QL homepage is at \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +The S3QL homepage is at \url{https://bitbucket.org/nikratio/s3ql/}. The full S3QL documentation should also be installed somewhere on your -system, common locations are \code{/usr/share/doc/s3ql} or -\code{/usr/local/doc/s3ql}. +system, common locations are \sphinxcode{/usr/share/doc/s3ql} or +\sphinxcode{/usr/local/doc/s3ql}. -\section{The \textbf{pcp} command} +\section{The \textbf{\texttt{pcp}} command} \label{man/pcp:the-command-command}\label{man/pcp::doc} \subsection{Synopsis} @@ -3816,19 +3849,19 @@ \subsection{Description} \label{man/pcp:description} -The \textbf{pcp} command is a is a wrapper that starts several -\textbf{sync} processes to copy directory trees in parallel. This is +The \textbf{\texttt{pcp}} command is a is a wrapper that starts several +\textbf{\texttt{sync}} processes to copy directory trees in parallel. This is allows much better copying performance on file system that have relatively high latency when retrieving individual files like S3QL. \textbf{Note}: Using this program only improves performance when copying \emph{from} an S3QL file system. When copying \emph{to} an S3QL file system, -using \textbf{pcp} is more likely to \emph{decrease} performance. +using \textbf{\texttt{pcp}} is more likely to \emph{decrease} performance. \subsection{Options} \label{man/pcp:options} -The \textbf{pcp} command accepts the following options: +The \textbf{\texttt{pcp}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}quiet] @@ -3837,11 +3870,11 @@ Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}version] just print program version and exit \item [-a] @@ -3854,7 +3887,7 @@ \subsection{Exit Codes} \label{man/pcp:exit-codes} -\textbf{pcp} may terminate with the following exit codes: +\textbf{\texttt{pcp}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -3871,10 +3904,10 @@ \subsection{See Also} \label{man/pcp:see-also} -\textbf{pcp} is shipped as part of S3QL, \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +\textbf{\texttt{pcp}} is shipped as part of S3QL, \url{https://bitbucket.org/nikratio/s3ql/}. -\section{The \textbf{expire\_backups} command} +\section{The \textbf{\texttt{expire\_backups}} command} \label{man/expire_backups:the-command-command}\label{man/expire_backups::doc} \subsection{Synopsis} @@ -3886,11 +3919,11 @@ \subsection{Description} \label{man/expire_backups:description} -The \textbf{expire\_backups} command intelligently remove old backups that are no +The \textbf{\texttt{expire\_backups}} command intelligently remove old backups that are no longer needed. To define what backups you want to keep for how long, you define a -number of \emph{age ranges}. \textbf{expire\_backups} ensures that you +number of \emph{age ranges}. \textbf{\texttt{expire\_backups}} ensures that you will have at least one backup in each age range at all times. It will keep exactly as many backups as are required for that and delete any backups that become redundant. @@ -3899,8 +3932,8 @@ of backup cycles. Every time you create a new backup, the existing backups age by one cycle. -Example: when \textbf{expire\_backups} is called with the age range -definition \code{1 3 7 14 31}, it will guarantee that you always have the +Example: when \textbf{\texttt{expire\_backups}} is called with the age range +definition \sphinxcode{1 3 7 14 31}, it will guarantee that you always have the following backups available: \begin{enumerate} \item {} @@ -3932,9 +3965,9 @@ cycles avoids these sort of problems. \end{notice} -\textbf{expire\_backups} usage is simple. It requires backups to be -stored in directories of the form \code{year-month-day\_hour:minute:seconds} -(\code{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current +\textbf{\texttt{expire\_backups}} usage is simple. It requires backups to be +stored in directories of the form \sphinxcode{year-month-day\_hour:minute:seconds} +(\sphinxcode{YYYY-MM-DD\_HH:mm:ss}) and works on all backups in the current directory. So for the above backup strategy, the correct invocation would be: @@ -3943,17 +3976,17 @@ \end{Verbatim} When storing your backups on an S3QL file system, you probably want to -specify the \code{-{-}use-s3qlrm} option as well. This tells -\textbf{expire\_backups} to use the {\hyperref[special:s3qlrm]{\emph{s3qlrm}}} command to +specify the \sphinxcode{-{-}use-s3qlrm} option as well. This tells +\textbf{\texttt{expire\_backups}} to use the {\hyperref[special:s3qlrm]{\sphinxcrossref{\DUrole{std,std-ref}{s3qlrm}}}} command to delete directories. -\textbf{expire\_backups} uses a ``state file'' to keep track which +\textbf{\texttt{expire\_backups}} uses a ``state file'' to keep track which backups are how many cycles old (since this cannot be inferred from the dates contained in the directory names). The standard name for -this state file is \code{.expire\_backups.dat}. If this file gets -damaged or deleted, \textbf{expire\_backups} no longer knows the ages +this state file is \sphinxcode{.expire\_backups.dat}. If this file gets +damaged or deleted, \textbf{\texttt{expire\_backups}} no longer knows the ages of the backups and refuses to work. In this case you can use the -\code{-{-}reconstruct-state} option to try to reconstruct the state +\sphinxcode{-{-}reconstruct-state} option to try to reconstruct the state from the backup dates. However, the accuracy of this reconstruction depends strongly on how rigorous you have been with making backups (it is only completely correct if the time between subsequent backups has @@ -3963,7 +3996,7 @@ \subsection{Options} \label{man/expire_backups:options} -The \textbf{expire\_backups} command accepts the following options: +The \textbf{\texttt{expire\_backups}} command accepts the following options: \begin{quote} \begin{optionlist}{3cm} \item [-{-}quiet] @@ -3972,11 +4005,11 @@ Activate debugging output from specified modules (use commas to separate multiple modules). Debug messages will be written to the target specified by the -\code{-{-}log} option. +\sphinxcode{-{-}log} option. \item [-{-}debug] Activate debugging output from all S3QL modules. Debug messages will be written to the target specified by -the \code{-{-}log} option. +the \sphinxcode{-{-}log} option. \item [-{-}version] just print program version and exit \item [-{-}state \textless{}file\textgreater{}] @@ -3988,14 +4021,14 @@ Try to reconstruct a missing state file from backup dates. \item [-{-}use-s3qlrm] -Use \code{s3qlrm} command to delete backups. +Use \sphinxcode{s3qlrm} command to delete backups. \end{optionlist} \end{quote} \subsection{Exit Codes} \label{man/expire_backups:exit-codes} -\textbf{expire\_backups} may terminate with the following exit codes: +\textbf{\texttt{expire\_backups}} may terminate with the following exit codes: \begin{quote}\begin{description} \item[{0}] \leavevmode Everything went well. @@ -4012,11 +4045,11 @@ \subsection{See Also} \label{man/expire_backups:see-also} -\textbf{expire\_backups} is shipped as part of S3QL, \href{https://bitbucket.org/nikratio/s3ql/}{https://bitbucket.org/nikratio/s3ql/}. +\textbf{\texttt{expire\_backups}} is shipped as part of S3QL, \url{https://bitbucket.org/nikratio/s3ql/}. \chapter{Further Resources / Getting Help} -\label{resources:resources}\label{resources:further-resources-getting-help}\label{resources::doc} +\label{resources:resources}\label{resources::doc}\label{resources:further-resources-getting-help} If you have questions or problems with S3QL that you weren't able to resolve with this manual, you might want to consider the following other resources: \begin{itemize} @@ -4037,7 +4070,7 @@ \chapter{Implementation Details} -\label{impl_details:impl-details}\label{impl_details:implementation-details}\label{impl_details::doc} +\label{impl_details:impl-details}\label{impl_details::doc}\label{impl_details:implementation-details} This section provides some background information on how S3QL works internally. Reading this section is not necessary to use S3QL. @@ -4062,7 +4095,7 @@ \item {} An S3QL filesystem can only be mounted on one computer at a time, -using a single \textbf{mount.s3ql} process. Otherwise changes made in +using a single \textbf{\texttt{mount.s3ql}} process. Otherwise changes made in one mountpoint will invariably be overwritten when the second mount point is unmounted. @@ -4083,7 +4116,8 @@ While the file system is mounted, blocks are cached locally. Blocks can also be compressed and encrypted before they are stored in -the storage backend. +the storage backend. This happens during upload, i.e. the cached data +is unencrypted and uncompressed. If some files have blocks with identical contents, the blocks will be stored in the same backend object (i.e., the data is only stored @@ -4124,7 +4158,7 @@ Amazon S3 object is uploaded and immediately downloaded again, the downloaded data might not yet reflect the changes done in the upload (see also -\href{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538}{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538}) +\url{http://developer.amazonwebservices.com/connect/message.jspa?messageID=38538}) For the data blocks this is not a problem because a data blocks always get a new object ID when they are updated. @@ -4144,8 +4178,8 @@ \section{Encryption} \label{impl_details:encryption} -When the file system is created, \textbf{mkfs.s3ql} generates a 256 bit -master key by reading from \code{/dev/random}. The master key is +When the file system is created, \textbf{\texttt{mkfs.s3ql}} generates a 256 bit +master key by reading from \sphinxcode{/dev/random}. The master key is encrypted with the passphrase that is entered by the user, and then stored with the rest of the file system data. Since the passphrase is only used to access the master key (which is used to encrypt the @@ -4161,8 +4195,8 @@ Once the session key has been calculated, a SHA256 HMAC is calculated over the data that is to be uploaded. Afterwards, the data is -compressed (unless \code{-{-}compress none} was passed to -\textbf{mount.s3ql}) and the HMAC inserted at the beginning. Both HMAC +compressed (unless \sphinxcode{-{-}compress none} was passed to +\textbf{\texttt{mount.s3ql}}) and the HMAC inserted at the beginning. Both HMAC and compressed data are then encrypted using 256 bit AES in CTR mode using \href{http://www.pycrypto.org/}{PyCrypto}. Finally, the nonce is inserted in front of the encrypted data and HMAC, and the packet is diff -Nru s3ql-2.22+dfsg/doc/latex/manual.toc s3ql-2.23+dfsg/doc/latex/manual.toc --- s3ql-2.22+dfsg/doc/latex/manual.toc 2017-06-23 16:36:56.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/manual.toc 2017-08-18 18:11:57.000000000 +0000 @@ -13,7 +13,7 @@ \contentsline {section}{\numberline {3.1}Google Storage}{7}{section.3.1} \contentsline {section}{\numberline {3.2}Amazon S3}{8}{section.3.2} \contentsline {section}{\numberline {3.3}OpenStack/Swift}{9}{section.3.3} -\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{9}{section.3.4} +\contentsline {section}{\numberline {3.4}Rackspace CloudFiles}{10}{section.3.4} \contentsline {section}{\numberline {3.5}S3 compatible}{10}{section.3.5} \contentsline {section}{\numberline {3.6}Local}{11}{section.3.6} \contentsline {chapter}{\numberline {4}Important Rules to Avoid Losing Data}{13}{chapter.4} @@ -30,11 +30,11 @@ \contentsline {chapter}{\numberline {7}Mounting}{21}{chapter.7} \contentsline {section}{\numberline {7.1}Permission Checking}{22}{section.7.1} \contentsline {section}{\numberline {7.2}Compression Algorithms}{22}{section.7.2} -\contentsline {section}{\numberline {7.3}Notes about Caching}{22}{section.7.3} +\contentsline {section}{\numberline {7.3}Notes about Caching}{23}{section.7.3} \contentsline {subsection}{\numberline {7.3.1}Maximum Number of Cache Entries}{23}{subsection.7.3.1} \contentsline {subsection}{\numberline {7.3.2}Cache Flushing and Expiration}{23}{subsection.7.3.2} \contentsline {section}{\numberline {7.4}Failure Modes}{23}{section.7.4} -\contentsline {section}{\numberline {7.5}Automatic Mounting}{23}{section.7.5} +\contentsline {section}{\numberline {7.5}Automatic Mounting}{24}{section.7.5} \contentsline {chapter}{\numberline {8}Advanced S3QL Features}{25}{chapter.8} \contentsline {section}{\numberline {8.1}Snapshotting and Copy-on-Write}{25}{section.8.1} \contentsline {subsection}{\numberline {8.1.1}Snapshotting vs Hardlinking}{25}{subsection.8.1.1} @@ -60,88 +60,88 @@ \contentsline {section}{\numberline {13.3}Improving copy performance}{39}{section.13.3} \contentsline {chapter}{\numberline {14}Known Issues}{41}{chapter.14} \contentsline {chapter}{\numberline {15}Manpages}{43}{chapter.15} -\contentsline {section}{\numberline {15.1}The \textbf {mkfs.s3ql} command}{43}{section.15.1} +\contentsline {section}{\numberline {15.1}The \textbf {\texttt {mkfs.s3ql}} command}{43}{section.15.1} \contentsline {subsection}{\numberline {15.1.1}Synopsis}{43}{subsection.15.1.1} \contentsline {subsection}{\numberline {15.1.2}Description}{43}{subsection.15.1.2} \contentsline {subsection}{\numberline {15.1.3}Options}{43}{subsection.15.1.3} \contentsline {subsection}{\numberline {15.1.4}Exit Codes}{44}{subsection.15.1.4} \contentsline {subsection}{\numberline {15.1.5}See Also}{44}{subsection.15.1.5} -\contentsline {section}{\numberline {15.2}The \textbf {s3qladm} command}{44}{section.15.2} +\contentsline {section}{\numberline {15.2}The \textbf {\texttt {s3qladm}} command}{44}{section.15.2} \contentsline {subsection}{\numberline {15.2.1}Synopsis}{44}{subsection.15.2.1} \contentsline {subsection}{\numberline {15.2.2}Description}{45}{subsection.15.2.2} \contentsline {subsection}{\numberline {15.2.3}Options}{45}{subsection.15.2.3} \contentsline {subsection}{\numberline {15.2.4}Actions}{45}{subsection.15.2.4} \contentsline {subsection}{\numberline {15.2.5}Exit Codes}{45}{subsection.15.2.5} \contentsline {subsection}{\numberline {15.2.6}See Also}{46}{subsection.15.2.6} -\contentsline {section}{\numberline {15.3}The \textbf {mount.s3ql} command}{46}{section.15.3} +\contentsline {section}{\numberline {15.3}The \textbf {\texttt {mount.s3ql}} command}{46}{section.15.3} \contentsline {subsection}{\numberline {15.3.1}Synopsis}{46}{subsection.15.3.1} \contentsline {subsection}{\numberline {15.3.2}Description}{46}{subsection.15.3.2} \contentsline {subsection}{\numberline {15.3.3}Options}{46}{subsection.15.3.3} \contentsline {subsection}{\numberline {15.3.4}Exit Codes}{47}{subsection.15.3.4} \contentsline {subsection}{\numberline {15.3.5}See Also}{48}{subsection.15.3.5} -\contentsline {section}{\numberline {15.4}The \textbf {s3qlstat} command}{48}{section.15.4} +\contentsline {section}{\numberline {15.4}The \textbf {\texttt {s3qlstat}} command}{48}{section.15.4} \contentsline {subsection}{\numberline {15.4.1}Synopsis}{48}{subsection.15.4.1} \contentsline {subsection}{\numberline {15.4.2}Description}{48}{subsection.15.4.2} \contentsline {subsection}{\numberline {15.4.3}Options}{49}{subsection.15.4.3} \contentsline {subsection}{\numberline {15.4.4}Exit Codes}{49}{subsection.15.4.4} \contentsline {subsection}{\numberline {15.4.5}See Also}{49}{subsection.15.4.5} -\contentsline {section}{\numberline {15.5}The \textbf {s3qlctrl} command}{49}{section.15.5} +\contentsline {section}{\numberline {15.5}The \textbf {\texttt {s3qlctrl}} command}{49}{section.15.5} \contentsline {subsection}{\numberline {15.5.1}Synopsis}{49}{subsection.15.5.1} \contentsline {subsection}{\numberline {15.5.2}Description}{49}{subsection.15.5.2} \contentsline {subsection}{\numberline {15.5.3}Options}{50}{subsection.15.5.3} \contentsline {subsection}{\numberline {15.5.4}Exit Codes}{50}{subsection.15.5.4} \contentsline {subsection}{\numberline {15.5.5}See Also}{50}{subsection.15.5.5} -\contentsline {section}{\numberline {15.6}The \textbf {s3qlcp} command}{50}{section.15.6} +\contentsline {section}{\numberline {15.6}The \textbf {\texttt {s3qlcp}} command}{50}{section.15.6} \contentsline {subsection}{\numberline {15.6.1}Synopsis}{50}{subsection.15.6.1} \contentsline {subsection}{\numberline {15.6.2}Description}{51}{subsection.15.6.2} -\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.21} +\contentsline {subsubsection}{Snapshotting vs Hardlinking}{51}{subsubsection*.22} \contentsline {subsection}{\numberline {15.6.3}Options}{51}{subsection.15.6.3} \contentsline {subsection}{\numberline {15.6.4}Exit Codes}{51}{subsection.15.6.4} \contentsline {subsection}{\numberline {15.6.5}See Also}{52}{subsection.15.6.5} -\contentsline {section}{\numberline {15.7}The \textbf {s3qlrm} command}{52}{section.15.7} +\contentsline {section}{\numberline {15.7}The \textbf {\texttt {s3qlrm}} command}{52}{section.15.7} \contentsline {subsection}{\numberline {15.7.1}Synopsis}{52}{subsection.15.7.1} \contentsline {subsection}{\numberline {15.7.2}Description}{52}{subsection.15.7.2} \contentsline {subsection}{\numberline {15.7.3}Options}{52}{subsection.15.7.3} \contentsline {subsection}{\numberline {15.7.4}Exit Codes}{52}{subsection.15.7.4} \contentsline {subsection}{\numberline {15.7.5}See Also}{53}{subsection.15.7.5} -\contentsline {section}{\numberline {15.8}The \textbf {s3qllock} command}{53}{section.15.8} +\contentsline {section}{\numberline {15.8}The \textbf {\texttt {s3qllock}} command}{53}{section.15.8} \contentsline {subsection}{\numberline {15.8.1}Synopsis}{53}{subsection.15.8.1} \contentsline {subsection}{\numberline {15.8.2}Description}{53}{subsection.15.8.2} \contentsline {subsection}{\numberline {15.8.3}Rationale}{53}{subsection.15.8.3} \contentsline {subsection}{\numberline {15.8.4}Options}{53}{subsection.15.8.4} \contentsline {subsection}{\numberline {15.8.5}Exit Codes}{54}{subsection.15.8.5} \contentsline {subsection}{\numberline {15.8.6}See Also}{54}{subsection.15.8.6} -\contentsline {section}{\numberline {15.9}The \textbf {umount.s3ql} command}{54}{section.15.9} +\contentsline {section}{\numberline {15.9}The \textbf {\texttt {umount.s3ql}} command}{54}{section.15.9} \contentsline {subsection}{\numberline {15.9.1}Synopsis}{54}{subsection.15.9.1} \contentsline {subsection}{\numberline {15.9.2}Description}{54}{subsection.15.9.2} \contentsline {subsection}{\numberline {15.9.3}Options}{54}{subsection.15.9.3} \contentsline {subsection}{\numberline {15.9.4}Exit Codes}{55}{subsection.15.9.4} \contentsline {subsection}{\numberline {15.9.5}See Also}{55}{subsection.15.9.5} -\contentsline {section}{\numberline {15.10}The \textbf {fsck.s3ql} command}{55}{section.15.10} +\contentsline {section}{\numberline {15.10}The \textbf {\texttt {fsck.s3ql}} command}{55}{section.15.10} \contentsline {subsection}{\numberline {15.10.1}Synopsis}{55}{subsection.15.10.1} \contentsline {subsection}{\numberline {15.10.2}Description}{55}{subsection.15.10.2} \contentsline {subsection}{\numberline {15.10.3}Options}{55}{subsection.15.10.3} \contentsline {subsection}{\numberline {15.10.4}Exit Codes}{56}{subsection.15.10.4} \contentsline {subsection}{\numberline {15.10.5}See Also}{57}{subsection.15.10.5} -\contentsline {section}{\numberline {15.11}The \textbf {s3ql\_oauth\_client} command}{57}{section.15.11} +\contentsline {section}{\numberline {15.11}The \textbf {\texttt {s3ql\_oauth\_client}} command}{57}{section.15.11} \contentsline {subsection}{\numberline {15.11.1}Synopsis}{57}{subsection.15.11.1} \contentsline {subsection}{\numberline {15.11.2}Description}{57}{subsection.15.11.2} \contentsline {subsection}{\numberline {15.11.3}Options}{57}{subsection.15.11.3} \contentsline {subsection}{\numberline {15.11.4}Exit Codes}{57}{subsection.15.11.4} \contentsline {subsection}{\numberline {15.11.5}See Also}{58}{subsection.15.11.5} -\contentsline {section}{\numberline {15.12}The \textbf {s3ql\_verify} command}{58}{section.15.12} +\contentsline {section}{\numberline {15.12}The \textbf {\texttt {s3ql\_verify}} command}{58}{section.15.12} \contentsline {subsection}{\numberline {15.12.1}Synopsis}{58}{subsection.15.12.1} \contentsline {subsection}{\numberline {15.12.2}Description}{58}{subsection.15.12.2} \contentsline {subsection}{\numberline {15.12.3}Options}{58}{subsection.15.12.3} \contentsline {subsection}{\numberline {15.12.4}Exit Codes}{59}{subsection.15.12.4} \contentsline {subsection}{\numberline {15.12.5}See Also}{59}{subsection.15.12.5} -\contentsline {section}{\numberline {15.13}The \textbf {pcp} command}{59}{section.15.13} +\contentsline {section}{\numberline {15.13}The \textbf {\texttt {pcp}} command}{59}{section.15.13} \contentsline {subsection}{\numberline {15.13.1}Synopsis}{59}{subsection.15.13.1} \contentsline {subsection}{\numberline {15.13.2}Description}{59}{subsection.15.13.2} \contentsline {subsection}{\numberline {15.13.3}Options}{60}{subsection.15.13.3} \contentsline {subsection}{\numberline {15.13.4}Exit Codes}{60}{subsection.15.13.4} \contentsline {subsection}{\numberline {15.13.5}See Also}{60}{subsection.15.13.5} -\contentsline {section}{\numberline {15.14}The \textbf {expire\_backups} command}{60}{section.15.14} +\contentsline {section}{\numberline {15.14}The \textbf {\texttt {expire\_backups}} command}{60}{section.15.14} \contentsline {subsection}{\numberline {15.14.1}Synopsis}{60}{subsection.15.14.1} \contentsline {subsection}{\numberline {15.14.2}Description}{60}{subsection.15.14.2} \contentsline {subsection}{\numberline {15.14.3}Options}{61}{subsection.15.14.3} @@ -151,7 +151,7 @@ \contentsline {chapter}{\numberline {17}Implementation Details}{65}{chapter.17} \contentsline {section}{\numberline {17.1}Metadata Storage}{65}{section.17.1} \contentsline {section}{\numberline {17.2}Data Storage}{65}{section.17.2} -\contentsline {section}{\numberline {17.3}Data De-Duplication}{65}{section.17.3} +\contentsline {section}{\numberline {17.3}Data De-Duplication}{66}{section.17.3} \contentsline {section}{\numberline {17.4}Caching}{66}{section.17.4} \contentsline {section}{\numberline {17.5}Eventual Consistency Handling}{66}{section.17.5} \contentsline {section}{\numberline {17.6}Encryption}{66}{section.17.6} diff -Nru s3ql-2.22+dfsg/doc/latex/sphinxhowto.cls s3ql-2.23+dfsg/doc/latex/sphinxhowto.cls --- s3ql-2.22+dfsg/doc/latex/sphinxhowto.cls 2014-05-03 03:52:31.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/sphinxhowto.cls 2016-11-23 14:06:39.000000000 +0000 @@ -5,6 +5,12 @@ \NeedsTeXFormat{LaTeX2e}[1995/12/01] \ProvidesClass{sphinxhowto}[2009/06/02 Document class (Sphinx HOWTO)] +\ifx\directlua\undefined\else +% if compiling with lualatex 0.85 or later load compatibility patch issued by +% the LaTeX team for older packages relying on \pdf named primitives. + \IfFileExists{luatex85.sty}{\RequirePackage{luatex85}}{} +\fi + % 'oneside' option overriding the 'twoside' default \newif\if@oneside \DeclareOption{oneside}{\@onesidetrue} @@ -29,30 +35,31 @@ % Change the title page to look a bit better, and fit in with the fncychap % ``Bjarne'' style a bit better. % -\renewcommand{\maketitle}{ - \rule{\textwidth}{1pt} +\renewcommand{\maketitle}{% + \noindent\rule{\textwidth}{1pt}\ifsphinxpdfoutput\newline\null\fi\par \ifsphinxpdfoutput \begingroup % These \defs are required to deal with multi-line authors; it % changes \\ to ', ' (comma-space), making it pass muster for % generating document info in the PDF file. - \def\\{, } - \def\and{and } + \def\\{, }% + \def\and{and }% \pdfinfo{ /Author (\@author) /Title (\@title) - } + }% \endgroup \fi \begin{flushright} - \sphinxlogo% - {\rm\Huge\py@HeaderFamily \@title} \par - {\em\large\py@HeaderFamily \py@release\releaseinfo} \par + \sphinxlogo + \py@HeaderFamily + {\Huge \@title }\par + {\itshape\large \py@release \releaseinfo}\par \vspace{25pt} - {\Large\py@HeaderFamily + {\Large \begin{tabular}[t]{c} \@author - \end{tabular}} \par + \end{tabular}}\par \vspace{25pt} \@date \par \py@authoraddress \par diff -Nru s3ql-2.22+dfsg/doc/latex/sphinxmanual.cls s3ql-2.23+dfsg/doc/latex/sphinxmanual.cls --- s3ql-2.22+dfsg/doc/latex/sphinxmanual.cls 2014-05-03 03:52:31.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/sphinxmanual.cls 2016-11-23 14:06:39.000000000 +0000 @@ -5,6 +5,12 @@ \NeedsTeXFormat{LaTeX2e}[1995/12/01] \ProvidesClass{sphinxmanual}[2009/06/02 Document class (Sphinx manual)] +\ifx\directlua\undefined\else +% if compiling with lualatex 0.85 or later load compatibility patch issued by +% the LaTeX team for older packages relying on \pdf named primitives. + \IfFileExists{luatex85.sty}{\RequirePackage{luatex85}}{} +\fi + % chapters starting at odd pages (overridden by 'openany' document option) \PassOptionsToClass{openright}{\sphinxdocclass} @@ -37,26 +43,27 @@ \begin{titlepage}% \let\footnotesize\small \let\footnoterule\relax - \rule{\textwidth}{1pt}% + \noindent\rule{\textwidth}{1pt}\ifsphinxpdfoutput\newline\null\fi\par \ifsphinxpdfoutput \begingroup % These \defs are required to deal with multi-line authors; it % changes \\ to ', ' (comma-space), making it pass muster for % generating document info in the PDF file. - \def\\{, } - \def\and{and } + \def\\{, }% + \def\and{and }% \pdfinfo{ /Author (\@author) /Title (\@title) - } + }% \endgroup \fi \begin{flushright}% - \sphinxlogo% - {\rm\Huge\py@HeaderFamily \@title \par}% - {\em\LARGE\py@HeaderFamily \py@release\releaseinfo \par} + \sphinxlogo + \py@HeaderFamily + {\Huge \@title \par} + {\itshape\LARGE \py@release\releaseinfo \par} \vfill - {\LARGE\py@HeaderFamily + {\LARGE \begin{tabular}[t]{c} \@author \end{tabular} @@ -70,52 +77,27 @@ \end{flushright}%\par \@thanks \end{titlepage}% - \cleardoublepage% \setcounter{footnote}{0}% \let\thanks\relax\let\maketitle\relax %\gdef\@thanks{}\gdef\@author{}\gdef\@title{} } - -% Catch the end of the {abstract} environment, but here make sure the abstract -% is followed by a blank page if the 'openright' option is used. -% -\let\py@OldEndAbstract=\endabstract -\renewcommand{\endabstract}{ - \if@openright - \ifodd\value{page} - \typeout{Adding blank page after the abstract.} - \vfil\pagebreak - \fi - \fi - \py@OldEndAbstract -} - -% This wraps the \tableofcontents macro with all the magic to get the spacing -% right and have the right number of pages if the 'openright' option has been -% used. This eliminates a fair amount of crud in the individual document files. -% \let\py@OldTableofcontents=\tableofcontents \renewcommand{\tableofcontents}{% + % before resetting page counter, let's do the right thing. + \if@openright\cleardoublepage\else\clearpage\fi \pagenumbering{roman}% - \setcounter{page}{1}% - \pagebreak% \pagestyle{plain}% - {% - \parskip = 0mm% - \py@OldTableofcontents% - \if@openright% - \ifodd\value{page}% - \typeout{Adding blank page after the table of contents.}% - \pagebreak\hspace{0pt}% - \fi% - \fi% - \cleardoublepage% - }% + \begingroup + \parskip \z@skip + \py@OldTableofcontents + \endgroup + % before resetting page counter, let's do the right thing. + \if@openright\cleardoublepage\else\clearpage\fi \pagenumbering{arabic}% - \@ifundefined{fancyhf}{}{\pagestyle{normal}}% + \ifdefined\fancyhf\pagestyle{normal}\fi } -\pagenumbering{alph} +\pagenumbering{alph}% avoid hyperref "duplicate destination" warnings % This is needed to get the width of the section # area wide enough in the % library reference. Doing it here keeps it the same for all the manuals. @@ -128,7 +110,7 @@ % For a report document class this environment is a chapter. \let\py@OldThebibliography=\thebibliography \renewcommand{\thebibliography}[1]{ - \cleardoublepage + \if@openright\cleardoublepage\else\clearpage\fi \phantomsection \py@OldThebibliography{1} \addcontentsline{toc}{chapter}{\bibname} @@ -140,7 +122,7 @@ \@ifclassloaded{memoir}{}{ \let\py@OldTheindex=\theindex \renewcommand{\theindex}{ - \cleardoublepage + \if@openright\cleardoublepage\else\clearpage\fi \phantomsection \py@OldTheindex \addcontentsline{toc}{chapter}{\indexname} diff -Nru s3ql-2.22+dfsg/doc/latex/sphinx.sty s3ql-2.23+dfsg/doc/latex/sphinx.sty --- s3ql-2.22+dfsg/doc/latex/sphinx.sty 2014-05-03 03:52:31.000000000 +0000 +++ s3ql-2.23+dfsg/doc/latex/sphinx.sty 2016-11-23 14:06:39.000000000 +0000 @@ -8,17 +8,42 @@ \NeedsTeXFormat{LaTeX2e}[1995/12/01] \ProvidesPackage{sphinx}[2010/01/15 LaTeX package (Sphinx markup)] +% this is the \ltx@ifundefined of ltxcmds.sty, which is loaded by +% hyperref.sty, but we need it before, and initial ltxcmds.sty +% as in TL2009/Debian had wrong definition. +\newcommand{\spx@ifundefined}[1]{% + \ifcsname #1\endcsname + \expandafter\ifx\csname #1\endcsname\relax + \expandafter\expandafter\expandafter\@firstoftwo + \else + \expandafter\expandafter\expandafter\@secondoftwo + \fi + \else + \expandafter\@firstoftwo + \fi +} + \@ifclassloaded{memoir}{}{\RequirePackage{fancyhdr}} +% for \text macro and \iffirstchoice@ conditional even if amsmath not loaded +\RequirePackage{amstext} \RequirePackage{textcomp} +% fancybox not used anymore and will be removed at Sphinx-1.5 \RequirePackage{fancybox} \RequirePackage{titlesec} \RequirePackage{tabulary} -\RequirePackage{amsmath} % for \text \RequirePackage{makeidx} +% For framing code-blocks and warning type notices, and shadowing topics \RequirePackage{framed} +\newif\ifspx@inframed % flag set if we are in a framed environment +% ifthen not used anymore and will be removed at Sphinx-1.5 \RequirePackage{ifthen} -\RequirePackage{color} +% The xcolor package draws better fcolorboxes around verbatim code +\IfFileExists{xcolor.sty}{ + \RequirePackage{xcolor} +}{ + \RequirePackage{color} +} % For highlighted code. \RequirePackage{fancyvrb} % For table captions. @@ -34,15 +59,18 @@ \RequirePackage{alltt} % Display "real" single quotes in literal blocks. \RequirePackage{upquote} +% For the H specifier. Do not \restylefloat{figure}, it breaks Sphinx code +% for allowing figures in tables. +\RequirePackage{float} % Redefine these colors to your liking in the preamble. \definecolor{TitleColor}{rgb}{0.126,0.263,0.361} \definecolor{InnerLinkColor}{rgb}{0.208,0.374,0.486} \definecolor{OuterLinkColor}{rgb}{0.216,0.439,0.388} -% Redefine these colors to something not white if you want to have colored +% Redefine these colors to something if you want to have colored % background and border for code examples. \definecolor{VerbatimColor}{rgb}{1,1,1} -\definecolor{VerbatimBorderColor}{rgb}{1,1,1} +\definecolor{VerbatimBorderColor}{rgb}{0,0,0} % Uncomment these two lines to ignore the paper size and make the page % size more like a typical published manual. @@ -51,31 +79,27 @@ %\renewcommand{\paperwidth}{7in} % O'Reilly ``Programmming Python'' % use pdfoutput for pTeX and dvipdfmx +% when pTeX (\kanjiskip is defined), set pdfoutput to evade \include{pdfcolor} \ifx\kanjiskip\undefined\else - \ifx\Gin@driver{dvipdfmx.def}\undefined\else - \newcount\pdfoutput\pdfoutput=0 - \fi + \newcount\pdfoutput\pdfoutput=0 \fi -% For graphicx, check if we are compiling under latex or pdflatex. -\ifx\pdftexversion\undefined - \usepackage{graphicx} -\else - \usepackage[pdftex]{graphicx} -\fi +\RequirePackage{graphicx} % for PDF output, use colors and maximal compression -\newif\ifsphinxpdfoutput\sphinxpdfoutputfalse -\ifx\pdfoutput\undefined\else\ifcase\pdfoutput +\newif\ifsphinxpdfoutput % used in \maketitle +\ifx\pdfoutput\undefined\else + \ifnum\pdfoutput=\z@ \let\py@NormalColor\relax \let\py@TitleColor\relax -\else + \else \sphinxpdfoutputtrue \input{pdfcolor} \def\py@NormalColor{\color[rgb]{0.0,0.0,0.0}} \def\py@TitleColor{\color{TitleColor}} \pdfcompresslevel=9 -\fi\fi + \fi +\fi % XeLaTeX can do colors, too \ifx\XeTeXrevision\undefined\else @@ -110,9 +134,10 @@ % Use this to set the font family for headers and other decor: \newcommand{\py@HeaderFamily}{\sffamily\bfseries} +\newcommand{\sphinxSetHeaderFamily}[1]{\renewcommand{\py@HeaderFamily}{#1}} % Redefine the 'normal' header/footer style when using "fancyhdr" package: -\@ifundefined{fancyhf}{}{ +\spx@ifundefined{fancyhf}{}{ % Use \pagestyle{normal} as the primary pagestyle for text. \fancypagestyle{normal}{ \fancyhf{} @@ -123,9 +148,8 @@ \renewcommand{\headrulewidth}{0.4pt} \renewcommand{\footrulewidth}{0.4pt} % define chaptermark with \@chappos when \@chappos is available for Japanese - \ifx\@chappos\undefined\else - \def\chaptermark##1{\markboth{\@chapapp\space\thechapter\space\@chappos\space ##1}{}} - \fi + \spx@ifundefined{@chappos}{} + {\def\chaptermark##1{\markboth{\@chapapp\space\thechapter\space\@chappos\space ##1}{}}} } % Update the plain style so we get the page number & footer line, % but not a chapter or section title. This is to keep the first @@ -139,54 +163,326 @@ } % Some custom font markup commands. -% -\newcommand{\strong}[1]{{\textbf{#1}}} -\newcommand{\code}[1]{\texttt{#1}} -\newcommand{\bfcode}[1]{\code{\bfseries#1}} -\newcommand{\email}[1]{\textsf{#1}} +% *** the macros without \sphinx prefix are still defined at bottom of file *** +\newcommand{\sphinxstrong}[1]{{\textbf{#1}}} +% let \sphinxcode and \sphinxbfcode use straight quotes. \@noligs patched by upquote, +% but needs protection in "moving arguments" such as for captions. +% Use \scantokens to handle e.g. \item[{\sphinxcode{'fontenc'}}] +\DeclareRobustCommand{\sphinxcode}[1]{{\@noligs\scantokens{\texttt{#1}\relax}}} +\newcommand{\sphinxbfcode}[1]{\sphinxcode{\bfseries#1}} +\newcommand{\sphinxemail}[1]{\textsf{#1}} +\newcommand{\sphinxtablecontinued}[1]{\textsf{#1}} +\newcommand{\sphinxtitleref}[1]{\emph{#1}} +\newcommand{\sphinxmenuselection}[1]{\emph{#1}} +\newcommand{\sphinxaccelerator}[1]{\underline{#1}} +\newcommand{\sphinxcrossref}[1]{\emph{#1}} +\newcommand{\sphinxtermref}[1]{\emph{#1}} + +% miscellaneous related to footnotes +\newcommand*{\sphinxAtStartFootnote}{\mbox{ }} +% Support large numbered footnotes in minipage (cf. admonitions) +\def\thempfootnote{\arabic{mpfootnote}} -% Redefine the Verbatim environment to allow border and background colors. +% Redefine the Verbatim environment to allow border and background colors +% and to handle the top caption in a non separable by pagebreak way. % The original environment is still used for verbatims within tables. \let\OriginalVerbatim=\Verbatim \let\endOriginalVerbatim=\endVerbatim -% Play with vspace to be able to keep the indentation. -\newlength\distancetoright -\def\mycolorbox#1{% - \setlength\distancetoright{\linewidth}% - \advance\distancetoright -\@totalleftmargin % - \fcolorbox{VerbatimBorderColor}{VerbatimColor}{% - \begin{minipage}{\distancetoright}% - #1 - \end{minipage}% - }% +\newcommand\spx@colorbox [2]{% +% #1 will be \fcolorbox or, for first part of frame: \spx@fcolorbox +% let the framing obey the current indentation (adapted from framed.sty's code). + \hskip\@totalleftmargin + \hskip-\fboxsep\hskip-\fboxrule + #1{VerbatimBorderColor}{VerbatimColor}{#2}% + \hskip-\fboxsep\hskip-\fboxrule + \hskip-\linewidth \hskip-\@totalleftmargin \hskip\columnwidth +} +% use of \color@b@x here is compatible with both xcolor.sty and color.sty +\def\spx@fcolorbox #1#2% + {\color@b@x {\fboxsep\z@\color{#1}\spx@VerbatimFBox}{\color{#2}}}% + +% The title is specified from outside as macro \sphinxVerbatimTitle. +% \sphinxVerbatimTitle is reset to empty after each use of Verbatim. +\newcommand*\sphinxVerbatimTitle {} +% Holder macro for labels of literal blocks. Set-up by LaTeX writer. +\newcommand*\sphinxLiteralBlockLabel {} +\newcommand*\sphinxSetupCaptionForVerbatim [2] +{% + \needspace{\sphinxliteralblockneedspace}% +% insert a \label via \sphinxLiteralBlockLabel +% reset to normal the color for the literal block caption +% the caption inserts \abovecaptionskip whitespace above itself (usually 10pt) +% there is also \belowcaptionskip but it is usually zero, hence the \smallskip + \def\sphinxVerbatimTitle + {\py@NormalColor\captionof{#1}{\sphinxLiteralBlockLabel #2}\smallskip }% +} + +% Inspired and adapted from framed.sty's \CustomFBox with extra handling +% of a non separable by pagebreak caption, and controlled counter stepping. +\newif\ifspx@myfirstframedpass +\long\def\spx@VerbatimFBox#1{% + \leavevmode + \begingroup + % framed.sty does some measuring but this macro adds possibly a caption + % use amsmath conditional to inhibit the caption counter stepping after + % first pass + \ifspx@myfirstframedpass\else\firstchoice@false\fi + \setbox\@tempboxa\hbox{\kern\fboxsep{#1}\kern\fboxsep}% + \hbox + {\lower\dimexpr\fboxrule+\fboxsep+\dp\@tempboxa + \hbox{% + \vbox{\ifx\sphinxVerbatimTitle\empty\else + % add the caption in a centered way above possibly indented frame + % hide its width from framed.sty's measuring step + % note that the caption brings \abovecaptionskip top vertical space + \moveright\dimexpr\fboxrule+.5\wd\@tempboxa + \hb@xt@\z@{\hss\begin{minipage}{\wd\@tempboxa}% + \sphinxVerbatimTitle + \end{minipage}\hss}\fi + % draw frame border _latest_ to avoid pdf viewer issue + \kern\fboxrule + \hbox{\kern\fboxrule + \vbox{\vskip\fboxsep\copy\@tempboxa\vskip\fboxsep}% + \kern-\wd\@tempboxa\kern-\fboxrule + \vrule\@width\fboxrule + \kern\wd\@tempboxa + \vrule\@width\fboxrule}% + \kern-\dimexpr\fboxsep+\ht\@tempboxa+\dp\@tempboxa + +\fboxsep+\fboxrule\relax + \hrule\@height\fboxrule + \kern\dimexpr\fboxsep+\ht\@tempboxa+\dp\@tempboxa+\fboxsep\relax + \hrule\@height\fboxrule}% + }}% + \endgroup + \global\spx@myfirstframedpassfalse +} + +% For linebreaks inside Verbatim environment from package fancyvrb. +\newbox\sphinxcontinuationbox +\newbox\sphinxvisiblespacebox +% These are user customizable e.g. from latex_elements's preamble key. +% Use of \textvisiblespace for compatibility with XeTeX/LuaTeX/fontspec. +\newcommand*\sphinxvisiblespace {\textcolor{red}{\textvisiblespace}} +\newcommand*\sphinxcontinuationsymbol {\textcolor{red}{\llap{\tiny$\m@th\hookrightarrow$}}} +\newcommand*\sphinxcontinuationindent {3ex } +\newcommand*\sphinxafterbreak {\kern\sphinxcontinuationindent\copy\sphinxcontinuationbox} + +% Take advantage of the already applied Pygments mark-up to insert +% potential linebreaks for TeX processing. +% {, <, #, %, $, ' and ": go to next line. +% _, }, ^, &, >, - and ~: stay at end of broken line. +% Use of \textquotesingle for straight quote. +\newcommand*\sphinxbreaksatspecials {% + \def\PYGZus{\discretionary{\char`\_}{\sphinxafterbreak}{\char`\_}}% + \def\PYGZob{\discretionary{}{\sphinxafterbreak\char`\{}{\char`\{}}% + \def\PYGZcb{\discretionary{\char`\}}{\sphinxafterbreak}{\char`\}}}% + \def\PYGZca{\discretionary{\char`\^}{\sphinxafterbreak}{\char`\^}}% + \def\PYGZam{\discretionary{\char`\&}{\sphinxafterbreak}{\char`\&}}% + \def\PYGZlt{\discretionary{}{\sphinxafterbreak\char`\<}{\char`\<}}% + \def\PYGZgt{\discretionary{\char`\>}{\sphinxafterbreak}{\char`\>}}% + \def\PYGZsh{\discretionary{}{\sphinxafterbreak\char`\#}{\char`\#}}% + \def\PYGZpc{\discretionary{}{\sphinxafterbreak\char`\%}{\char`\%}}% + \def\PYGZdl{\discretionary{}{\sphinxafterbreak\char`\$}{\char`\$}}% + \def\PYGZhy{\discretionary{\char`\-}{\sphinxafterbreak}{\char`\-}}% + \def\PYGZsq{\discretionary{}{\sphinxafterbreak\textquotesingle}{\textquotesingle}}% + \def\PYGZdq{\discretionary{}{\sphinxafterbreak\char`\"}{\char`\"}}% + \def\PYGZti{\discretionary{\char`\~}{\sphinxafterbreak}{\char`\~}}% +} + +\def\sphinx@verbatim@nolig@list {\do \`}% +% Some characters . , ; ? ! / are not pygmentized. +% This macro makes them "active" and they will insert potential linebreaks +\newcommand*\sphinxbreaksatpunct {% + \lccode`\~`\.\lowercase{\def~}{\discretionary{\char`\.}{\sphinxafterbreak}{\char`\.}}% + \lccode`\~`\,\lowercase{\def~}{\discretionary{\char`\,}{\sphinxafterbreak}{\char`\,}}% + \lccode`\~`\;\lowercase{\def~}{\discretionary{\char`\;}{\sphinxafterbreak}{\char`\;}}% + \lccode`\~`\:\lowercase{\def~}{\discretionary{\char`\:}{\sphinxafterbreak}{\char`\:}}% + \lccode`\~`\?\lowercase{\def~}{\discretionary{\char`\?}{\sphinxafterbreak}{\char`\?}}% + \lccode`\~`\!\lowercase{\def~}{\discretionary{\char`\!}{\sphinxafterbreak}{\char`\!}}% + \lccode`\~`\/\lowercase{\def~}{\discretionary{\char`\/}{\sphinxafterbreak}{\char`\/}}% + \catcode`\.\active + \catcode`\,\active + \catcode`\;\active + \catcode`\:\active + \catcode`\?\active + \catcode`\!\active + \catcode`\/\active + \lccode`\~`\~ } -\def\FrameCommand{\mycolorbox} \renewcommand{\Verbatim}[1][1]{% + % quit horizontal mode if we are still in a paragraph + \par % list starts new par, but we don't want it to be set apart vertically - \bgroup\parskip=0pt% - \smallskip% - % The list environement is needed to control perfectly the vertical - % space. - \list{}{% - \setlength\parskip{0pt}% - \setlength\itemsep{0ex}% - \setlength\topsep{0ex}% - \setlength\partopsep{0pt}% - \setlength\leftmargin{0pt}% - }% - \item\MakeFramed {\FrameRestore}% - \small% - \OriginalVerbatim[#1]% + \parskip\z@skip + % first, let's check if there is a caption + \ifx\sphinxVerbatimTitle\empty + \addvspace\z@% counteract possible previous negative skip (French lists!) + \smallskip + % there was no caption. Check if nevertheless a label was set. + \ifx\sphinxLiteralBlockLabel\empty\else + % we require some space to be sure hyperlink target from \phantomsection + % will not be separated from upcoming verbatim by a page break + \needspace{\sphinxliteralblockwithoutcaptionneedspace}% + \phantomsection\sphinxLiteralBlockLabel + \fi + \fi + % non-empty \sphinxVerbatimTitle has label inside it (in case there is one) + % Customize framed.sty \MakeFramed to glue caption to literal block + \global\spx@myfirstframedpasstrue + % via \spx@fcolorbox, will use \spx@VerbatimFBox which inserts title + \def\FrameCommand {\spx@colorbox\spx@fcolorbox }% + \let\FirstFrameCommand\FrameCommand + % for mid pages and last page portion of (long) split frame: + \def\MidFrameCommand{\spx@colorbox\fcolorbox }% + \let\LastFrameCommand\MidFrameCommand + % fancyvrb's Verbatim puts each input line in (unbreakable) horizontal boxes. + % This customization wraps each line from the input in a \vtop, thus + % allowing it to wrap and display on two or more lines in the latex output. + % - The codeline counter will be increased only once. + % - The wrapped material will not break across pages, it is impossible + % to achieve this without extensive rewrite of fancyvrb. + % - The (not used in sphinx) obeytabs option to Verbatim is + % broken by this change (showtabs and tabspace work). + \sbox\sphinxcontinuationbox {\sphinxcontinuationsymbol}% + \sbox\sphinxvisiblespacebox {\FV@SetupFont\sphinxvisiblespace}% + \def\FancyVerbFormatLine ##1{\hsize\linewidth + \vtop{\raggedright\hyphenpenalty\z@\exhyphenpenalty\z@ + \doublehyphendemerits\z@\finalhyphendemerits\z@ + \strut ##1\strut}% + }% + % If the linebreak is at a space, the latter will be displayed as visible + % space at end of first line, and a continuation symbol starts next line. + % Stretch/shrink are however usually zero for typewriter font. + \def\FV@Space {% + \nobreak\hskip\z@ plus\fontdimen3\font minus\fontdimen4\font + \discretionary{\copy\sphinxvisiblespacebox}{\sphinxafterbreak} + {\kern\fontdimen2\font}% + }% + % go around fancyvrb's check of @currenvir (for case of minipage below) + \renewcommand*{\VerbatimEnvironment}{\gdef\FV@EnvironName{Verbatim}}% + % go around fancyvrb's check of current list depth + \def\@toodeep {\advance\@listdepth\@ne}% + % Allow breaks at special characters using \PYG... macros. + \sphinxbreaksatspecials + % The list environment is needed to control perfectly the vertical space. + % Note: \OuterFrameSep used by framed.sty is later set to \topsep hence 0pt. + % - if caption: vertical space above caption = (\abovecaptionskip + D) with + % D = \baselineskip-\FrameHeightAdjust, and then \smallskip above frame. + % - if no caption: (\smallskip + D) above frame. By default D=6pt. + % Use trivlist rather than list to avoid possible "too deeply nested" error. + \itemsep \z@skip + \topsep \z@skip + \partopsep \z@skip% trivlist will set \parsep to \parskip = zero (see above) + % \leftmargin will be set to zero by trivlist + \rightmargin\z@ + \parindent \z@% becomes \itemindent. Default zero, but perhaps overwritten. + \trivlist\item\relax + % use a minipage if we are already inside a framed environment + \ifspx@inframed\noindent\begin{minipage}{\linewidth}\fi + \MakeFramed {% adapted over from framed.sty's snugshade environment + \advance\hsize-\width\@totalleftmargin\z@\linewidth\hsize + \@setminipage }% + \small + % For grid placement from \strut's in \FancyVerbFormatLine + \lineskip\z@skip + % Breaks at punctuation characters . , ; ? ! and / need catcode=\active + % and the active comma should not be overwritten by \@noligs + \let\verbatim@nolig@list \sphinx@verbatim@nolig@list + \OriginalVerbatim[#1,codes*=\sphinxbreaksatpunct]% } \renewcommand{\endVerbatim}{% - \endOriginalVerbatim% - \endMakeFramed% - \endlist% - % close group to restore \parskip - \egroup% -} + \endOriginalVerbatim + \par\unskip\@minipagefalse\endMakeFramed + \ifspx@inframed\end{minipage}\fi + \endtrivlist +} + +% define macro to frame contents and add shadow on right and bottom +% use public names for customizable lengths +\newlength\sphinxshadowsep \setlength\sphinxshadowsep {5pt} +\newlength\sphinxshadowsize \setlength\sphinxshadowsize {4pt} +\newlength\sphinxshadowrule +% this uses \fboxrule value at loading time of sphinx.sty (0.4pt normally) +\setlength\sphinxshadowrule {\fboxrule} + +\long\def\spx@ShadowFBox#1{% + \leavevmode\begingroup + % first we frame the box #1 + \setbox\@tempboxa + \hbox{\vrule\@width\sphinxshadowrule + \vbox{\hrule\@height\sphinxshadowrule + \kern\sphinxshadowsep + \hbox{\kern\sphinxshadowsep #1\kern\sphinxshadowsep}% + \kern\sphinxshadowsep + \hrule\@height\sphinxshadowrule}% + \vrule\@width\sphinxshadowrule}% + % Now we add the shadow, like \shadowbox from fancybox.sty would do + \dimen@\dimexpr.5\sphinxshadowrule+\sphinxshadowsize\relax + \hbox{\vbox{\offinterlineskip + \hbox{\copy\@tempboxa\kern-.5\sphinxshadowrule + % add shadow on right side + \lower\sphinxshadowsize + \hbox{\vrule\@height\ht\@tempboxa \@width\dimen@}% + }% + \kern-\dimen@ % shift back vertically to bottom of frame + % and add shadow at bottom + \moveright\sphinxshadowsize + \vbox{\hrule\@width\wd\@tempboxa \@height\dimen@}% + }% + % move left by the size of right shadow so shadow adds no width + \kern-\sphinxshadowsize + }% + \endgroup +} + +% use framed.sty to allow page breaks in frame+shadow +% works well inside Lists and Quote-like environments +% produced by ``topic'' directive (or local contents) +% could nest if LaTeX writer authorized it +\newenvironment{sphinxShadowBox} + {\def\FrameCommand {\spx@ShadowFBox }% + % configure framed.sty not to add extra vertical spacing + \spx@ifundefined{OuterFrameSep}{}{\OuterFrameSep\z@skip}% + % the \trivlist will add the vertical spacing on top and bottom which is + % typical of center environment as used in Sphinx <= 1.4.1 + % the \noindent has the effet of an extra blank line on top, to + % imitate closely the layout from Sphinx <= 1.4.1; the \FrameHeightAdjust + % will put top part of frame on this baseline. + \def\FrameHeightAdjust {\baselineskip}% + \trivlist\item\noindent + % use a minipage if we are already inside a framed environment + \ifspx@inframed\begin{minipage}{\linewidth}\fi + \MakeFramed {\spx@inframedtrue + % framed.sty puts into "\width" the added width (=2shadowsep+2shadowrule) + % adjust \hsize to what the contents must use + \advance\hsize-\width + % adjust LaTeX parameters to behave properly in indented/quoted contexts + \FrameRestore + % typeset the contents as in a minipage (Sphinx <= 1.4.1 used a minipage and + % itemize/enumerate are therein typeset more tightly, we want to keep + % that). We copy-paste from LaTeX source code but don't do a real minipage. + \@pboxswfalse + % for footnotes, but Sphinx inactivates footnotes in topics + \def\@mpfn{mpfootnote}\def\thempfn{\thempfootnote}\c@mpfootnote\z@ + \let\@footnotetext\@mpfootnotetext + \let\@listdepth\@mplistdepth \@mplistdepth\z@ + \@minipagerestore + \@setminipage + }% + }% + {% insert the "endminipage" code + \par\unskip + % handle (currently non existing) minipage style footnotes + \ifvoid\@mpfootins\else + \vskip\skip\@mpfootins\normalcolor\footnoterule\unvbox\@mpfootins + \fi + \@minipagefalse + \endMakeFramed + \ifspx@inframed\end{minipage}\fi + \endtrivlist + } % \moduleauthor{name}{email} @@ -203,8 +499,12 @@ {\py@TitleColor\thesubsection}{0.5em}{\py@TitleColor}{\py@NormalColor} \titleformat{\subsubsection}{\py@HeaderFamily}% {\py@TitleColor\thesubsubsection}{0.5em}{\py@TitleColor}{\py@NormalColor} -\titleformat{\paragraph}{\small\py@HeaderFamily}% - {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor} +% By default paragraphs (and subsubsections) will not be numbered because +% sphinxmanual.cls and sphinxhowto.cls set secnumdepth to 2 +\titleformat{\paragraph}{\py@HeaderFamily}% + {\py@TitleColor\theparagraph}{0.5em}{\py@TitleColor}{\py@NormalColor} +\titleformat{\subparagraph}{\py@HeaderFamily}% + {\py@TitleColor\thesubparagraph}{0.5em}{\py@TitleColor}{\py@NormalColor} % {fulllineitems} is the main environment for object descriptions. % @@ -221,25 +521,25 @@ }{\end{list}} % \optional is used for ``[, arg]``, i.e. desc_optional nodes. -\newcommand{\optional}[1]{% +\newcommand{\sphinxoptional}[1]{% {\textnormal{\Large[}}{#1}\hspace{0.5mm}{\textnormal{\Large]}}} \newlength{\py@argswidth} \newcommand{\py@sigparams}[2]{% - \parbox[t]{\py@argswidth}{#1\code{)}#2}} + \parbox[t]{\py@argswidth}{#1\sphinxcode{)}#2}} \newcommand{\pysigline}[1]{\item[#1]\nopagebreak} \newcommand{\pysiglinewithargsret}[3]{% - \settowidth{\py@argswidth}{#1\code{(}}% + \settowidth{\py@argswidth}{#1\sphinxcode{(}}% \addtolength{\py@argswidth}{-2\py@argswidth}% \addtolength{\py@argswidth}{\linewidth}% - \item[#1\code{(}\py@sigparams{#2}{#3}]} + \item[#1\sphinxcode{(}\py@sigparams{#2}{#3}]} % Production lists % \newenvironment{productionlist}{ -% \def\optional##1{{\Large[}##1{\Large]}} - \def\production##1##2{\\\code{##1}&::=&\code{##2}} - \def\productioncont##1{\\& &\code{##1}} +% \def\sphinxoptional##1{{\Large[}##1{\Large]}} + \def\production##1##2{\\\sphinxcode{##1}&::=&\sphinxcode{##2}} + \def\productioncont##1{\\& &\sphinxcode{##1}} \parindent=2em \indent \setlength{\LTpre}{0pt} @@ -251,34 +551,69 @@ % Notices / Admonitions % -\newlength{\py@noticelength} -\newcommand{\py@heavybox}{ - \setlength{\fboxrule}{1pt} - \setlength{\fboxsep}{6pt} - \setlength{\py@noticelength}{\linewidth} - \addtolength{\py@noticelength}{-2\fboxsep} - \addtolength{\py@noticelength}{-2\fboxrule} - %\setlength{\shadowsize}{3pt} - \noindent\Sbox - \minipage{\py@noticelength} -} -\newcommand{\py@endheavybox}{ - \endminipage - \endSbox - \fbox{\TheSbox} -} - -\newcommand{\py@lightbox}{{% - \setlength\parskip{0pt}\par - \noindent\rule[0ex]{\linewidth}{0.5pt}% - \par\noindent\vspace{-0.5ex}% - }} -\newcommand{\py@endlightbox}{{% - \setlength{\parskip}{0pt}% - \par\noindent\rule[0.5ex]{\linewidth}{0.5pt}% - \par\vspace{-0.5ex}% - }} +% Code adapted from framed.sty's "snugshade" environment. +% Nesting works (inner frames do not allow page breaks). +\newcommand{\py@heavybox}{\par + \setlength{\FrameRule}{\p@}% 1pt + \setlength{\FrameSep}{\dimexpr.6\baselineskip-\FrameRule\relax} + % configure framed.sty's parameters to obtain same vertical spacing + % as for "light" boxes. We need for this to manually insert parskip glue and + % revert a skip done by framed before the frame. + \spx@ifundefined{OuterFrameSep}{}{\OuterFrameSep\z@skip}% + \vspace{\FrameHeightAdjust} + % copied/adapted from framed.sty's snugshade + \def\FrameCommand##1{\hskip\@totalleftmargin + \fboxsep\FrameSep \fboxrule\FrameRule\fbox{##1}% + \hskip-\linewidth \hskip-\@totalleftmargin \hskip\columnwidth}% + % use a minipage if we are already inside a framed environment + \ifspx@inframed + \noindent\begin{minipage}{\linewidth} + \else + % handle case where notice is first thing in a list item (or is quoted) + \if@inlabel + \noindent\par\vspace{-\baselineskip} + \else + \vspace{\parskip} + \fi + \fi + \MakeFramed {\spx@inframedtrue + \advance\hsize-\width \@totalleftmargin\z@ \linewidth\hsize + % minipage initialization copied from LaTeX source code. + \@pboxswfalse + % for footnotes + \def\@mpfn{mpfootnote}\def\thempfn{\thempfootnote}\c@mpfootnote\z@ + \let\@footnotetext\@mpfootnotetext + \let\@listdepth\@mplistdepth \@mplistdepth\z@ + \@minipagerestore + \@setminipage }% + } +\newcommand{\py@endheavybox}{% + \par\unskip + % handles footnotes + \ifvoid\@mpfootins\else + \vskip\skip\@mpfootins\normalcolor\footnoterule\unvbox\@mpfootins + \fi + \@minipagefalse + \endMakeFramed + \ifspx@inframed\end{minipage}\fi + % arrange for similar spacing below frame as for "light" boxes. + \vskip .4\baselineskip + } + +\newcommand{\py@lightbox}{% + \par\allowbreak + \noindent\rule{\linewidth}{0.5pt}\par\nobreak + {\parskip\z@skip\noindent}% + } +\newcommand{\py@endlightbox}{% + \par + % counteract previous possible negative skip (French lists!): + % (we can't cancel that any earlier \vskip introduced a potential pagebreak) + \ifdim\lastskip<\z@\vskip-\lastskip\fi + \nobreak\vbox{\noindent\kern\@totalleftmargin + \rule[.4\baselineskip]{\linewidth}{0.5pt}\hss}\allowbreak + } % Some are quite plain: \newcommand{\py@noticestart@note}{\py@lightbox} @@ -305,7 +640,7 @@ \newenvironment{notice}[2]{ \def\py@noticetype{#1} \csname py@noticestart@#1\endcsname - \strong{#2} + \sphinxstrong{#2} % <- legacy code creates a space after {#2} }{\csname py@noticeend@\py@noticetype\endcsname} % Allow the release number to be specified independently of the @@ -335,7 +670,7 @@ % This sets up the fancy chapter headings that make the documents look % at least a little better than the usual LaTeX output. % -\@ifundefined{ChTitleVar}{}{ +\spx@ifundefined{ChTitleVar}{}{ \ChNameVar{\raggedleft\normalsize\py@HeaderFamily} \ChNumVar{\raggedleft \bfseries\Large\py@HeaderFamily} \ChTitleVar{\raggedleft \textrm{\Huge\py@HeaderFamily}} @@ -375,7 +710,7 @@ % The following is stuff copied from docutils' latex writer. % -\newcommand{\optionlistlabel}[1]{\bf #1 \hfill} +\newcommand{\optionlistlabel}[1]{\normalfont\bfseries #1 \hfill}% \bf deprecated \newenvironment{optionlist}[1] {\begin{list}{} {\setlength{\labelwidth}{#1} @@ -397,30 +732,36 @@ \raggedright} {\end{list}} -% Redefine includgraphics for avoiding images larger than the screen size -% If the size is not specified. +% Re-define \includegraphics to resize images larger than the line width +% if the size is not specified. +% Warning: future version of Sphinx will not modify original \includegraphics, +% Below custom code will be direct definition of \sphinxincludegraphics, with +% \py@Oldincludegraphics replaced by direct use of original \includegraphics. \let\py@Oldincludegraphics\includegraphics - -\newbox\image@box% -\newdimen\image@width% -\renewcommand\includegraphics[2][\@empty]{% - \ifx#1\@empty% - \setbox\image@box=\hbox{\py@Oldincludegraphics{#2}}% - \image@width\wd\image@box% - \ifdim \image@width>\linewidth% - \setbox\image@box=\hbox{\py@Oldincludegraphics[width=\linewidth]{#2}}% - \box\image@box% - \else% - \py@Oldincludegraphics{#2}% - \fi% - \else% +\newbox\spx@image@box +\renewcommand*{\includegraphics}[2][\@empty]{% + \ifx\@empty #1% attention, #1 could be bb.., bad if first after \ifx + \setbox\spx@image@box=\hbox{\py@Oldincludegraphics{#2}}% + \ifdim \wd\spx@image@box>\linewidth + \py@Oldincludegraphics[width=\linewidth]{#2}% + \else + \leavevmode\box\spx@image@box + \fi + \else \py@Oldincludegraphics[#1]{#2}% - \fi% + \fi } +% Writer will put \sphinxincludegraphics in LaTeX source, and with this, +% documents which used their own modified \includegraphics will compile +% as before. But see warning above. +\newcommand*{\sphinxincludegraphics}{\includegraphics} % to make pdf with correct encoded bookmarks in Japanese % this should precede the hyperref package -\ifx\kanjiskip\undefined\else +\ifx\kanjiskip\undefined +% for non-Japanese: make sure bookmarks are ok also with lualatex + \PassOptionsToPackage{pdfencoding=unicode}{hyperref} +\else \usepackage{atbegshi} \ifx\ucs\undefined \ifnum 42146=\euc"A4A2 @@ -443,24 +784,30 @@ % \capstart for figures that actually have a caption.) \RequirePackage{hypcap} +% Set up styles of URL: it should be placed after hyperref +\urlstyle{same} + % From docutils.writers.latex2e -\providecommand{\DUspan}[2]{% - {% group ("span") to limit the scope of styling commands - \@for\node@class@name:=#1\do{% - \ifcsname docutilsrole\node@class@name\endcsname% - \csname docutilsrole\node@class@name\endcsname% +% inline markup (custom roles) +% \DUrole{#1}{#2} tries \DUrole#1{#2} +\providecommand*{\DUrole}[2]{% + \ifcsname DUrole#1\endcsname% + \csname DUrole#1\endcsname{#2}% + \else% backwards compatibility: try \docutilsrole#1{#2} + \ifcsname docutilsrole#1\endcsname% + \csname docutilsrole#1\endcsname{#2}% + \else% + #2% \fi% - }% - {#2}% node content - }% close "span" + \fi% } -\providecommand*{\DUprovidelength}[2]{ - \ifthenelse{\isundefined{#1}}{\newlength{#1}\setlength{#1}{#2}}{} +\providecommand*{\DUprovidelength}[2]{% + \ifdefined#1\else\newlength{#1}\setlength{#1}{#2}\fi } \DUprovidelength{\DUlineblockindent}{2.5em} -\ifthenelse{\isundefined{\DUlineblock}}{ +\ifdefined\DUlineblock\else \newenvironment{DUlineblock}[1]{% \list{}{\setlength{\partopsep}{\parskip} \addtolength{\partopsep}{\baselineskip} @@ -471,8 +818,7 @@ \raggedright } {\endlist} -}{} - +\fi % From footmisc.sty: allows footnotes in titles \let\FN@sf@@footnote\footnote @@ -522,3 +868,50 @@ \gdef\@chappos{} } \fi + +% Define literal-block environment +\RequirePackage{newfloat} +\DeclareFloatingEnvironment{literal-block} +\spx@ifundefined{c@chapter} + {\SetupFloatingEnvironment{literal-block}{within=section,placement=h}} + {\SetupFloatingEnvironment{literal-block}{within=chapter,placement=h}} +\SetupFloatingEnvironment{literal-block}{name=List} +% control caption around literal-block +\RequirePackage{capt-of} +\RequirePackage{needspace} +% if the left page space is less than \literalblockneedspace, insert page-break +\newcommand{\sphinxliteralblockneedspace}{5\baselineskip} +\newcommand{\sphinxliteralblockwithoutcaptionneedspace}{1.5\baselineskip} + +% figure in table +\newenvironment{sphinxfigure-in-table}[1][\linewidth]{% + \def\@captype{figure}% + \begin{minipage}{#1}% +}{\end{minipage}} +% store original \caption macro for use with figures in longtable and tabulary +\AtBeginDocument{\let\spx@originalcaption\caption} +\newcommand*\sphinxfigcaption + {\ifx\equation$%$% this is trick to identify tabulary first pass + \firstchoice@false\else\firstchoice@true\fi + \spx@originalcaption } + +% by default, also define macros with the no-prefix names +\ifsphinxKeepOldNames + \typeout{** (sphinx) defining (legacy) text style macros without \string\sphinx\space prefix} + \typeout{** if clashes with packages, set latex_keep_old_macro_names=False in conf.py} + \@for\@tempa:=strong,bfcode,email,tablecontinued,titleref,% + menuselection,accelerator,crossref,termref,optional\do +{% first, check if command with no prefix already exists + \expandafter\newcommand\csname\@tempa\endcsname{}% + % if no error give it the meaning defined so far with \sphinx prefix + \expandafter\let\csname\@tempa\expandafter\endcsname + \csname sphinx\@tempa\endcsname + % redefine the \sphinx prefixed macro to expand to non-prefixed one + \expandafter\def\csname sphinx\@tempa\expandafter\endcsname + \expandafter{\csname\@tempa\endcsname}% +} + % robustified case needs special treatment + \newcommand\code{}\let\code\relax + \DeclareRobustCommand{\code}[1]{{\@noligs\scantokens{\texttt{#1}\relax}}} + \def\sphinxcode{\code}% +\fi diff -Nru s3ql-2.22+dfsg/doc/man/fsck.s3ql.1 s3ql-2.23+dfsg/doc/man/fsck.s3ql.1 --- s3ql-2.22+dfsg/doc/man/fsck.s3ql.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/fsck.s3ql.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "FSCK.S3QL" "1" "June 23, 2017" "2.22" "S3QL" +.TH "FSCK.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME fsck.s3ql \- Check an S3QL file system for errors . diff -Nru s3ql-2.22+dfsg/doc/man/mkfs.s3ql.1 s3ql-2.23+dfsg/doc/man/mkfs.s3ql.1 --- s3ql-2.22+dfsg/doc/man/mkfs.s3ql.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/mkfs.s3ql.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MKFS.S3QL" "1" "June 23, 2017" "2.22" "S3QL" +.TH "MKFS.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME mkfs.s3ql \- Create an S3QL file system . diff -Nru s3ql-2.22+dfsg/doc/man/mount.s3ql.1 s3ql-2.23+dfsg/doc/man/mount.s3ql.1 --- s3ql-2.22+dfsg/doc/man/mount.s3ql.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/mount.s3ql.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "MOUNT.S3QL" "1" "June 23, 2017" "2.22" "S3QL" +.TH "MOUNT.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME mount.s3ql \- Mount an S3QL file system . diff -Nru s3ql-2.22+dfsg/doc/man/s3qladm.1 s3ql-2.23+dfsg/doc/man/s3qladm.1 --- s3ql-2.22+dfsg/doc/man/s3qladm.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3qladm.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLADM" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QLADM" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3qladm \- Manage S3QL file systems . diff -Nru s3ql-2.22+dfsg/doc/man/s3qlcp.1 s3ql-2.23+dfsg/doc/man/s3qlcp.1 --- s3ql-2.22+dfsg/doc/man/s3qlcp.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3qlcp.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCP" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QLCP" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3qlcp \- Copy-on-write replication on S3QL file systems . diff -Nru s3ql-2.22+dfsg/doc/man/s3qlctrl.1 s3ql-2.23+dfsg/doc/man/s3qlctrl.1 --- s3ql-2.22+dfsg/doc/man/s3qlctrl.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3qlctrl.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLCTRL" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QLCTRL" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3qlctrl \- Control a mounted S3QL file system . diff -Nru s3ql-2.22+dfsg/doc/man/s3qllock.1 s3ql-2.23+dfsg/doc/man/s3qllock.1 --- s3ql-2.22+dfsg/doc/man/s3qllock.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3qllock.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLLOCK" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QLLOCK" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3qllock \- Make trees on an S3QL file system immutable . diff -Nru s3ql-2.22+dfsg/doc/man/s3ql_oauth_client.1 s3ql-2.23+dfsg/doc/man/s3ql_oauth_client.1 --- s3ql-2.22+dfsg/doc/man/s3ql_oauth_client.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3ql_oauth_client.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_OAUTH_CLIENT" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QL_OAUTH_CLIENT" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3ql_oauth_client \- Obtain Google Storage OAuth2 tokens . diff -Nru s3ql-2.22+dfsg/doc/man/s3qlrm.1 s3ql-2.23+dfsg/doc/man/s3qlrm.1 --- s3ql-2.22+dfsg/doc/man/s3qlrm.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3qlrm.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLRM" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QLRM" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3qlrm \- Fast tree removal on S3QL file systems . diff -Nru s3ql-2.22+dfsg/doc/man/s3qlstat.1 s3ql-2.23+dfsg/doc/man/s3qlstat.1 --- s3ql-2.22+dfsg/doc/man/s3qlstat.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3qlstat.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QLSTAT" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QLSTAT" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3qlstat \- Gather S3QL file system statistics . diff -Nru s3ql-2.22+dfsg/doc/man/s3ql_verify.1 s3ql-2.23+dfsg/doc/man/s3ql_verify.1 --- s3ql-2.22+dfsg/doc/man/s3ql_verify.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/s3ql_verify.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "S3QL_VERIFY" "1" "June 23, 2017" "2.22" "S3QL" +.TH "S3QL_VERIFY" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME s3ql_verify \- Verify data in an S3QL file system . diff -Nru s3ql-2.22+dfsg/doc/man/umount.s3ql.1 s3ql-2.23+dfsg/doc/man/umount.s3ql.1 --- s3ql-2.22+dfsg/doc/man/umount.s3ql.1 2017-06-23 16:36:53.000000000 +0000 +++ s3ql-2.23+dfsg/doc/man/umount.s3ql.1 2017-08-18 18:11:53.000000000 +0000 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "UMOUNT.S3QL" "1" "June 23, 2017" "2.22" "S3QL" +.TH "UMOUNT.S3QL" "1" "Aug 18, 2017" "2.23" "S3QL" .SH NAME umount.s3ql \- Unmount an S3QL file system . Binary files /tmp/tmpjEMYYj/Zh44knhT3p/s3ql-2.22+dfsg/doc/manual.pdf and /tmp/tmpjEMYYj/vhDMW5BgJF/s3ql-2.23+dfsg/doc/manual.pdf differ diff -Nru s3ql-2.22+dfsg/PKG-INFO s3ql-2.23+dfsg/PKG-INFO --- s3ql-2.22+dfsg/PKG-INFO 2017-06-23 16:36:59.000000000 +0000 +++ s3ql-2.23+dfsg/PKG-INFO 2017-08-18 18:11:57.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.22 +Version: 2.23 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -146,6 +146,12 @@ Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + Professional Support + ==================== + + Professional support is available. Please contact Nikolaus Rath + for details. + Contributing ============ diff -Nru s3ql-2.22+dfsg/README.rst s3ql-2.23+dfsg/README.rst --- s3ql-2.22+dfsg/README.rst 2016-08-08 23:51:01.000000000 +0000 +++ s3ql-2.23+dfsg/README.rst 2017-08-18 17:46:35.000000000 +0000 @@ -137,6 +137,12 @@ Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. +Professional Support +==================== + +Professional support is available. Please contact Nikolaus Rath + for details. + Contributing ============ diff -Nru s3ql-2.22+dfsg/rst/backends.rst s3ql-2.23+dfsg/rst/backends.rst --- s3ql-2.22+dfsg/rst/backends.rst 2017-06-15 22:46:18.000000000 +0000 +++ s3ql-2.23+dfsg/rst/backends.rst 2017-08-18 17:46:35.000000000 +0000 @@ -191,11 +191,11 @@ swift://[:]/[/] -for keystore (v2) authentication, the storage URL is :: +for Keystone (v2) authentication, the storage URL is :: swiftks://[:]/:[/] -Note that when using keystore authentication, you can (and have to) +Note that when using Keystone authentication, you can (and have to) specify the storage region of the container as well. In both cases, *hostname* name should be the name of the @@ -207,7 +207,7 @@ When using legacy authentication, the backend login and password correspond to the OpenStack username and API Access Key. When using -keystore authentication, the backend password is your regular +Keystone authentication, the backend password is your regular OpenStack password and the backend login combines you OpenStack username and tenant name in the form `:`. If no tenant is required, the OpenStack username alone may be used as backend @@ -245,10 +245,26 @@ performance as object data will be transmitted to the server more than once in some circumstances. +.. option:: no-feature-detection + + If this option is specified, S3QL does not try to dynamically detect + advanced features of the Swift backend. In this case S3QL can only + use the least common denominator of supported Swift versions and + configurations. + .. __: http://tools.ietf.org/html/rfc2616#section-8.2.3 .. _OpenStack: http://www.openstack.org/ .. _Swift: http://openstack.org/projects/storage/ +.. NOTE:: + + The Swift API unfortunately lacks a number of features that S3QL + normally makes use of. S3QL works around these deficiencies as much + as possible. However, this means that storing data using the Swift + backend generally requires more network round-trips and transfer + volume than the other backends. Also, S3QL requires Swift storage + servers to provide immediate consistency for newly created objects. + Rackspace CloudFiles ==================== @@ -273,19 +289,6 @@ The Rackspace backend accepts the same backend options as the :ref:`OpenStack backend `. -.. NOTE:: - - As of January 2012, Rackspace does not give any durability or - consistency guarantees (see :ref:`durability` for why this is - important). However, Rackspace support agents seem prone to claim - very high guarantees. Unless explicitly backed by their terms of - service, any such statement should thus be viewed with - suspicion. S3QL developers have also `repeatedly experienced - `_ - similar issues with the credibility and competence of the Rackspace - support. - - .. _Rackspace: http://www.rackspace.com/ diff -Nru s3ql-2.22+dfsg/rst/impl_details.rst s3ql-2.23+dfsg/rst/impl_details.rst --- s3ql-2.22+dfsg/rst/impl_details.rst 2016-08-08 23:51:01.000000000 +0000 +++ s3ql-2.23+dfsg/rst/impl_details.rst 2017-08-18 17:46:35.000000000 +0000 @@ -47,7 +47,8 @@ While the file system is mounted, blocks are cached locally. Blocks can also be compressed and encrypted before they are stored in -the storage backend. +the storage backend. This happens during upload, i.e. the cached data +is unencrypted and uncompressed. If some files have blocks with identical contents, the blocks will be stored in the same backend object (i.e., the data is only stored diff -Nru s3ql-2.22+dfsg/setup.cfg s3ql-2.23+dfsg/setup.cfg --- s3ql-2.22+dfsg/setup.cfg 2017-06-23 16:36:59.000000000 +0000 +++ s3ql-2.23+dfsg/setup.cfg 2017-08-18 18:11:57.000000000 +0000 @@ -2,7 +2,6 @@ allow_hosts = None [egg_info] -tag_date = 0 -tag_svn_revision = 0 tag_build = +tag_date = 0 diff -Nru s3ql-2.22+dfsg/src/s3ql/backends/s3c.py s3ql-2.23+dfsg/src/s3ql/backends/s3c.py --- s3ql-2.22+dfsg/src/s3ql/backends/s3c.py 2017-06-20 16:20:44.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/backends/s3c.py 2017-08-18 17:46:35.000000000 +0000 @@ -161,10 +161,12 @@ or exc.status == 408)): return True - # Temporary workaround for https://bitbucket.org/nikratio/s3ql/issues/87. - # We still need to find a proper string + # Temporary workaround for + # https://bitbucket.org/nikratio/s3ql/issues/87 and + # https://bitbucket.org/nikratio/s3ql/issues/252 elif (isinstance(exc, ssl.SSLError) and - str(exc).startswith('[SSL: BAD_WRITE_RETRY]')): + (str(exc).startswith('[SSL: BAD_WRITE_RETRY]') or + str(exc).startswith('[SSL: BAD_LENGTH]'))): return True return False @@ -230,7 +232,14 @@ log.debug('started with %s, %s', prefix, start_after) keys_remaining = True - marker = self.prefix + start_after + + # Without this, a call to list('foo') would result + # in *prefix* being longer than *marker* - which causes + # trouble for some S3 implementions (minio). + if start_after: + marker = self.prefix + start_after + else: + marker = '' prefix = self.prefix + prefix while keys_remaining: @@ -273,7 +282,7 @@ root.clear() except Exception as exc: - if is_temp_network_error(exc): + if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError): # We probably can't use the connection anymore self.conn.disconnect() raise @@ -730,7 +739,7 @@ return read_response() except Exception as exc: - if is_temp_network_error(exc): + if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError): # We probably can't use the connection anymore self.conn.disconnect() raise diff -Nru s3ql-2.22+dfsg/src/s3ql/backends/s3.py s3ql-2.23+dfsg/src/s3ql/backends/s3.py --- s3ql-2.22+dfsg/src/s3ql/backends/s3.py 2017-06-15 22:58:18.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/backends/s3.py 2017-08-18 17:46:35.000000000 +0000 @@ -56,7 +56,10 @@ if not re.match('^[a-z0-9][a-z0-9.-]{1,60}[a-z0-9]$', bucket_name): raise QuietError('Invalid bucket name.', exitcode=2) - hostname = 's3-%s.amazonaws.com' % self.region + if self.region == 'us-east-1': + hostname = 's3.amazonaws.com' + else: + hostname = 's3-%s.amazonaws.com' % self.region prefix = hit.group(3) or '' port = 443 if ssl_context else 80 diff -Nru s3ql-2.22+dfsg/src/s3ql/backends/swiftks.py s3ql-2.23+dfsg/src/s3ql/backends/swiftks.py --- s3ql-2.22+dfsg/src/s3ql/backends/swiftks.py 2016-08-08 23:51:01.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/backends/swiftks.py 2017-08-18 17:46:35.000000000 +0000 @@ -120,6 +120,8 @@ # fall through to scheme used for authentication pass + self._detect_features(o.hostname, o.port) + conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) diff -Nru s3ql-2.22+dfsg/src/s3ql/backends/swift.py s3ql-2.23+dfsg/src/s3ql/backends/swift.py --- s3ql-2.22+dfsg/src/s3ql/backends/swift.py 2016-10-25 17:21:39.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/backends/swift.py 2017-08-18 17:46:35.000000000 +0000 @@ -6,7 +6,7 @@ This work can be distributed under the terms of the GNU GPLv3. ''' -from ..logging import logging, QuietError # Ensure use of custom logger class +from ..logging import logging, QuietError, LOG_ONCE # Ensure use of custom logger class from .. import BUFSIZE from .common import (AbstractBackend, NoSuchObject, retry, AuthorizationError, DanglingStorageURLError, retry_generator, get_proxy, @@ -24,6 +24,7 @@ import os import urllib.parse import ssl +from distutils.version import LooseVersion log = logging.getLogger(__name__) @@ -38,7 +39,8 @@ """ hdr_prefix = 'X-Object-' - known_options = {'no-ssl', 'ssl-ca-path', 'tcp-timeout', 'disable-expect100'} + known_options = {'no-ssl', 'ssl-ca-path', 'tcp-timeout', + 'disable-expect100', 'no-feature-detection'} _add_meta_headers = s3c.Backend._add_meta_headers _extractmeta = s3c.Backend._extractmeta @@ -59,6 +61,7 @@ self.conn = None self.password = password self.login = login + self.features = Features() # We may need the context even if no-ssl has been specified, # because no-ssl applies only to the authentication URL. @@ -128,16 +131,19 @@ elif (isinstance(exc, HTTPError) and ((500 <= exc.status <= 599 and exc.status not in (501,505,508,510,511,523)) - or exc.status == 408)): + or exc.status == 408 + or 'client disconnected' in exc.msg.lower())): return True elif is_temp_network_error(exc): return True - # Temporary workaround for https://bitbucket.org/nikratio/s3ql/issues/87. - # We still need to find a proper string + # Temporary workaround for + # https://bitbucket.org/nikratio/s3ql/issues/87 and + # https://bitbucket.org/nikratio/s3ql/issues/252 elif (isinstance(exc, ssl.SSLError) and - str(exc).startswith('[SSL: BAD_WRITE_RETRY]')): + (str(exc).startswith('[SSL: BAD_WRITE_RETRY]') or + str(exc).startswith('[SSL: BAD_LENGTH]'))): return True return False @@ -189,6 +195,8 @@ # fall through to scheme used for authentication pass + self._detect_features(o.hostname, o.port) + conn = HTTPConnection(o.hostname, o.port, proxy=self.proxy, ssl_context=ssl_context) conn.timeout = int(self.options.get('tcp-timeout', 20)) @@ -231,7 +239,7 @@ try: resp = self._do_request_inner(method, path, body=body, headers=headers) except Exception as exc: - if is_temp_network_error(exc): + if is_temp_network_error(exc) or isinstance(exc, ssl.SSLError): # We probably can't use the connection anymore self.conn.disconnect() raise @@ -380,7 +388,7 @@ headers = CaseInsensitiveDict() if metadata is None: metadata = dict() - self._add_meta_headers(headers, metadata) + self._add_meta_headers(headers, metadata, chunksize=self.features.max_meta_len) return ObjectW(key, self, headers) @@ -401,10 +409,154 @@ elif exc.status != 404: raise - # We cannot wrap the entire copy() method into a retry() decorator, because - # copy() issues multiple requests. If the server happens to regularly close - # the connection after a request (even though it was processed correctly), - # we'd never make any progress if we always restart from the first request. + @retry + def _delete_multi(self, keys, force=False): + """Doing bulk delete of multiple objects at a time. + + This is a feature of the configurable middleware "Bulk" so it can only + be used after the middleware was detected. (Introduced in Swift 1.8.0.rc1) + + See https://docs.openstack.org/swift/latest/middleware.html#bulk-delete + and https://github.com/openstack/swift/blob/master/swift/common/middleware/bulk.py + + A request example: + 'POST /?bulk-delete + Content-type: text/plain; charset=utf-8 + Accept: application/json + + /container/prefix_key1 + /container/prefix_key2' + + A successful response: + 'HTTP/1.1 200 OK + Content-Type: application/json + + {"Number Not Found": 0, + "Response Status": "200 OK", + "Response Body": "", + "Errors": [], + "Number Deleted": 2}' + + An error response: + 'HTTP/1.1 200 OK + Content-Type: application/json + + {"Number Not Found": 0, + "Response Status": "500 Internal Server Error", + "Response Body": "An error description", + "Errors": [ + ['/container/prefix_key2', 'An error description'] + ], + "Number Deleted": 1}' + + Response when some objects where not found: + 'HTTP/1.1 200 OK + Content-Type: application/json + + {"Number Not Found": 1, + "Response Status": "400 Bad Request", + "Response Body": "Invalid bulk delete.", + "Errors": [], + "Number Deleted": 1}' + """ + + body = [] + esc_prefix = "/%s/%s" % (urllib.parse.quote(self.container_name), + urllib.parse.quote(self.prefix)) + for key in keys: + body.append('%s%s' % (esc_prefix, urllib.parse.quote(key))) + body = '\n'.join(body).encode('utf-8') + headers = { + 'content-type': 'text/plain; charset=utf-8', + 'accept': 'application/json' + } + + resp = self._do_request('POST', '/', subres='bulk-delete', body=body, headers=headers) + + # bulk deletes should always return 200 + if resp.status is not 200: + raise HTTPError(resp.status, resp.reason, resp.headers) + + hit = re.match('^application/json(;\s*charset="?(.+?)"?)?$', + resp.headers['content-type']) + if not hit: + log.error('Unexpected server response. Expected json, got:\n%s', + self._dump_response(resp)) + raise RuntimeError('Unexpected server reply') + + # there might be an arbitrary amount of whitespace before the + # JSON response (to keep the connection from timing out) + # but json.loads discards these whitespace characters automatically + resp_dict = json.loads(self.conn.readall().decode(hit.group(2) or 'utf-8')) + + hit = re.match('^([0-9]{3})', resp_dict['Response Status']) + if not hit: + log.error('Unexpected server response. Expected valid Response Status, got:\n%s', + resp_dict) + raise RuntimeError('Unexpected server reply') + resp_status = int(hit.group(1)) + + if resp_status is 200: + # No errors occured, everything has been deleted + del keys[:] + return + + # Some errors occured, so we need to determine what has + # been deleted and what hasn't + failed_keys = [] + offset = len(esc_prefix) + for error in resp_dict['Errors']: + fullkey = error[0] + # stangely the name is url encoded in JSON + assert fullkey.startswith(esc_prefix) + key = urllib.parse.unquote(fullkey[offset:]) + failed_keys.append(key) + log.debug('Delete %s failed with %s', key, error[1]) + for key in keys[:]: + if key not in failed_keys: + keys.remove(key) + + # If *force*, just modify the passed list and return without + # raising an exception, otherwise raise exception for the first error + if force: + return + + if resp_status in (400, 404) and len(resp_dict['Errors']) == 0: + # Swift returns 400 instead of 404 when files were not found. + # (but we also accept the correct status code 404 if Swift + # decides to correct this in the future) + + # ensure that we actually have objects that were not found + # (otherwise there is a logic error that we need to know about) + assert(resp_dict['Number Not Found'] > 0) + + # Unfortunately we cannot find out from the response which object + # was actually not found. + # Since AbstractBackend.delete_multi allows this, we just + # swallow this error even when *force* is False. + return + + raise HTTPError(resp_status, resp_dict['Response Body'], {}) + + @copy_ancestor_docstring + def delete_multi(self, keys, force=False): + log.debug('started with %s', keys) + + if self.features.has_bulk_delete: + while len(keys) > 0: + tmp = keys[:self.features.max_deletes] + try: + self._delete_multi(tmp, force=force) + finally: + keys[:self.features.max_deletes] = tmp + else: + super().delete_multi(keys, force=force) + + # We cannot wrap the entire _copy_via_put_post() method into a retry() + # decorator, because _copy_via_put_post() issues multiple requests. + # If the server happens to regularly close the connection after a request + # (even though it was processed correctly), we'd never make any progress + # if we always restart from the first request. # We experimented with adding a retry(fn, args, kwargs) function to wrap # individual calls, but this doesn't really improve the code because we # typically also have to wrap e.g. a discard() or assert_empty() call, so we @@ -415,12 +567,8 @@ self._do_request(method, path, headers=headers) self.conn.discard() - @copy_ancestor_docstring - def copy(self, src, dest, metadata=None): - log.debug('started with %s, %s', src, dest) - if dest.endswith(TEMP_SUFFIX) or src.endswith(TEMP_SUFFIX): - raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) - + def _copy_via_put_post(self, src, dest, metadata=None): + """Fallback copy method for older Swift implementations.""" headers = CaseInsensitiveDict() headers['X-Copy-From'] = '/%s/%s%s' % (self.container_name, self.prefix, src) @@ -445,7 +593,7 @@ # Update metadata headers = CaseInsensitiveDict() - self._add_meta_headers(headers, metadata) + self._add_meta_headers(headers, metadata, chunksize=self.features.max_meta_len) self._copy_helper('POST', '/%s%s' % (self.prefix, dest), headers) # Rename object @@ -454,11 +602,34 @@ self._copy_helper('PUT', '/%s%s' % (self.prefix, final_dest), headers) @retry + def _copy_via_copy(self, src, dest, metadata=None): + """Copy for more modern Swift implementations that know the + X-Fresh-Metadata option and the native COPY method.""" + headers = CaseInsensitiveDict() + headers['Destination'] = '/%s/%s%s' % (self.container_name, self.prefix, dest) + if metadata is not None: + self._add_meta_headers(headers, metadata, chunksize=self.features.max_meta_len) + headers['X-Fresh-Metadata'] = 'true' + resp = self._do_request('COPY', '/%s%s' % (self.prefix, src), headers=headers) + self._assert_empty_response(resp) + + @copy_ancestor_docstring + def copy(self, src, dest, metadata=None): + log.debug('started with %s, %s', src, dest) + if dest.endswith(TEMP_SUFFIX) or src.endswith(TEMP_SUFFIX): + raise ValueError('Keys must not end with %s' % TEMP_SUFFIX) + + if self.features.has_copy: + self._copy_via_copy(src, dest, metadata=metadata) + else: + self._copy_via_put_post(src, dest, metadata=metadata) + + @retry @copy_ancestor_docstring def update_meta(self, key, metadata): log.debug('started with %s', key) headers = CaseInsensitiveDict() - self._add_meta_headers(headers, metadata) + self._add_meta_headers(headers, metadata, chunksize=self.features.max_meta_len) self._do_request('POST', '/%s%s' % (self.prefix, key), headers=headers) self.conn.discard() @@ -516,6 +687,82 @@ def close(self): self.conn.disconnect() + def _detect_features(self, hostname, port): + '''Try to figure out the Swift version and supported features by + examining the /info endpoint of the storage server. + + See https://docs.openstack.org/swift/latest/middleware.html#discoverability + ''' + + if 'no-feature-detection' in self.options: + log.debug('Skip feature detection') + return + + ssl_context = self.ssl_context + if 'no-ssl' in self.options: + ssl_context = None + + if not port: + port = 443 if ssl_context else 80 + + detected_features = Features() + + with HTTPConnection(hostname, port, proxy=self.proxy, + ssl_context=ssl_context) as conn: + conn.timeout = int(self.options.get('tcp-timeout', 20)) + + log.debug('GET /info') + conn.send_request('GET', '/info') + resp = conn.read_response() + + # 200, 401, 403 and 404 are all OK since the /info endpoint + # may not be accessible (misconfiguration) or may not + # exist (old Swift version). + if resp.status not in (200, 401, 403, 404): + log.error("Wrong server response.\n%s", + self._dump_response(resp, body=conn.read(2048))) + raise HTTPError(resp.status, resp.reason, resp.headers) + + if resp.status is 200: + hit = re.match('^application/json(;\s*charset="?(.+?)"?)?$', + resp.headers['content-type']) + if not hit: + log.error("Wrong server response. Expected json. Got: \n%s", + self._dump_response(resp, body=conn.read(2048))) + raise RuntimeError('Unexpected server reply') + + info = json.loads(conn.readall().decode(hit.group(2) or 'utf-8')) + swift_info = info.get('swift', {}) + + log.debug('%s:%s/info returns %s', hostname, port, info) + + swift_version_string = swift_info.get('version', None) + if swift_version_string and \ + LooseVersion(swift_version_string) >= LooseVersion('2.8'): + detected_features.has_copy = True + + # Default metadata value length constrain is 256 bytes + # but the provider could configure another value. + # We only decrease the chunk size since 255 is a big enough chunk size. + max_meta_len = swift_info.get('max_meta_value_length', None) + if isinstance(max_meta_len, int) and max_meta_len < 256: + detected_features.max_meta_len = max_meta_len + + if info.get('bulk_delete', False): + detected_features.has_bulk_delete = True + # The block cache removal queue has a capacity of 1000. + # We do not need bigger values than that (the default maximum is 10000). + detected_features.max_deletes = max(1, min(1000, + int(info['bulk_delete'].get('max_deletes_per_request', 1000)))) + + log.info('Detected Swift features for %s:%s: %s', + hostname, port, detected_features, extra=LOG_ONCE) + else: + log.debug('%s:%s/info not found or not accessible. Skip feature detection.', + hostname, port) + + self.features = detected_features + class AuthenticationExpired(Exception): '''Raised if the provided Authentication Token has expired''' @@ -525,3 +772,43 @@ def __str__(self): return 'Auth token expired. Server said: %s' % self.msg + +class Features: + """Set of configurable features for Swift servers. + + Swift is deployed in many different versions and configurations. + To be able to use advanced features like bulk delete we need to + make sure that the Swift server we are using can handle them. + + This is a value object.""" + + __slots__ = ['has_copy', 'has_bulk_delete', 'max_deletes', 'max_meta_len'] + + def __init__(self, has_copy=False, has_bulk_delete=False, max_deletes=1000, + max_meta_len=255): + self.has_copy = has_copy + self.has_bulk_delete = has_bulk_delete + self.max_deletes = max_deletes + self.max_meta_len = max_meta_len + + def __str__(self): + features = [] + if self.has_copy: + features.append('copy via COPY') + if self.has_bulk_delete: + features.append('Bulk delete %d keys at a time' % self.max_deletes) + features.append('maximum meta value length is %d bytes' % self.max_meta_len) + return ', '.join(features) + + def __repr__(self): + init_kwargs = [p + '=' + repr(getattr(self, p)) for p in self.__slots__] + return 'Features(%s)' % ', '.join(init_kwargs) + + def __hash__(self): + return hash(repr(self)) + + def __eq__(self, other): + return repr(self) == repr(other) + + def __ne__(self, other): + return repr(self) != repr(other) diff -Nru s3ql-2.22+dfsg/src/s3ql/block_cache.py s3ql-2.23+dfsg/src/s3ql/block_cache.py --- s3ql-2.22+dfsg/src/s3ql/block_cache.py 2017-06-16 19:17:43.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/block_cache.py 2017-08-18 17:46:35.000000000 +0000 @@ -8,6 +8,7 @@ from . import BUFSIZE from .database import NoSuchRowError +from .backends.common import NoSuchObject from .multi_lock import MultiLock from .logging import logging # Ensure use of custom logger class from collections import OrderedDict @@ -279,6 +280,9 @@ if not os.path.exists(self.path): os.mkdir(self.path) + # Initialized fromt the outside to prevent cyclic dependency + self.fs = None + def __len__(self): '''Get number of objects in cache''' return len(self.cache) @@ -686,10 +690,14 @@ except QueueEmpty: tmp = FlushSentinel - if (tmp is FlushSentinel or tmp is QuitSentinel) and ids: + if tmp in (FlushSentinel,QuitSentinel) and ids: log.debug('removing: %s', ids) - with self.backend_pool() as backend: - backend.delete_multi(['s3ql_data_%d' % i for i in ids]) + try: + with self.backend_pool() as backend: + backend.delete_multi(['s3ql_data_%d' % i for i in ids]) + except NoSuchObject: + log.warning('Backend lost object s3ql_data_%d' % ids.pop(0)) + self.fs.failsafe = True ids = [] else: ids.append(tmp) @@ -697,7 +705,6 @@ if tmp is QuitSentinel: break - @contextmanager def get(self, inode, blockno): """Get file handle for block `blockno` of `inode` @@ -964,6 +971,9 @@ return (len(self.cache), used, dirty_cnt, dirty_size, remove_cnt) def __del__(self): + # break reference loop + self.fs = None + if len(self.cache) == 0: return diff -Nru s3ql-2.22+dfsg/src/s3ql/ctrl.py s3ql-2.23+dfsg/src/s3ql/ctrl.py --- s3ql-2.22+dfsg/src/s3ql/ctrl.py 2016-08-08 23:51:01.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/ctrl.py 2017-08-18 17:46:35.000000000 +0000 @@ -37,6 +37,7 @@ subparsers = parser.add_subparsers(metavar='', dest='action', help='may be either of') + subparsers.required = True subparsers.add_parser('flushcache', help='flush file system cache', parents=[pparser]) subparsers.add_parser('upload-meta', help='Upload metadata', diff -Nru s3ql-2.22+dfsg/src/s3ql/deltadump.c s3ql-2.23+dfsg/src/s3ql/deltadump.c --- s3ql-2.22+dfsg/src/s3ql/deltadump.c 2017-06-23 16:36:31.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/deltadump.c 2017-08-18 17:46:56.000000000 +0000 @@ -1,4 +1,4 @@ -/* Generated by Cython 0.24.0a0 */ +/* Generated by Cython 0.25.2 */ #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -7,7 +7,7 @@ #elif PY_VERSION_HEX < 0x02060000 || (0x03000000 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x03020000) #error Cython requires Python 2.6+ or Python 3.2+. #else -#define CYTHON_ABI "0_24_0a0" +#define CYTHON_ABI "0_25_2" #include #ifndef offsetof #define offsetof(type, member) ( (size_t) & ((type*)0) -> member ) @@ -29,6 +29,11 @@ #ifndef DL_EXPORT #define DL_EXPORT(t) t #endif +#ifndef HAVE_LONG_LONG + #if PY_VERSION_HEX >= 0x03030000 || (PY_MAJOR_VERSION == 2 && PY_VERSION_HEX >= 0x02070000) + #define HAVE_LONG_LONG + #endif +#endif #ifndef PY_LONG_LONG #define PY_LONG_LONG LONG_LONG #endif @@ -37,13 +42,110 @@ #endif #ifdef PYPY_VERSION #define CYTHON_COMPILING_IN_PYPY 1 + #define CYTHON_COMPILING_IN_PYSTON 0 + #define CYTHON_COMPILING_IN_CPYTHON 0 + #undef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 0 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #undef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #undef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 1 + #undef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 0 + #undef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 0 + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 +#elif defined(PYSTON_VERSION) + #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 1 #define CYTHON_COMPILING_IN_CPYTHON 0 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #undef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 0 + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #undef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 0 + #undef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 0 #else #define CYTHON_COMPILING_IN_PYPY 0 + #define CYTHON_COMPILING_IN_PYSTON 0 #define CYTHON_COMPILING_IN_CPYTHON 1 + #ifndef CYTHON_USE_TYPE_SLOTS + #define CYTHON_USE_TYPE_SLOTS 1 + #endif + #if PY_MAJOR_VERSION < 3 + #undef CYTHON_USE_ASYNC_SLOTS + #define CYTHON_USE_ASYNC_SLOTS 0 + #elif !defined(CYTHON_USE_ASYNC_SLOTS) + #define CYTHON_USE_ASYNC_SLOTS 1 + #endif + #if PY_VERSION_HEX < 0x02070000 + #undef CYTHON_USE_PYLONG_INTERNALS + #define CYTHON_USE_PYLONG_INTERNALS 0 + #elif !defined(CYTHON_USE_PYLONG_INTERNALS) + #define CYTHON_USE_PYLONG_INTERNALS 1 + #endif + #ifndef CYTHON_USE_PYLIST_INTERNALS + #define CYTHON_USE_PYLIST_INTERNALS 1 + #endif + #ifndef CYTHON_USE_UNICODE_INTERNALS + #define CYTHON_USE_UNICODE_INTERNALS 1 + #endif + #if PY_VERSION_HEX < 0x030300F0 + #undef CYTHON_USE_UNICODE_WRITER + #define CYTHON_USE_UNICODE_WRITER 0 + #elif !defined(CYTHON_USE_UNICODE_WRITER) + #define CYTHON_USE_UNICODE_WRITER 1 + #endif + #ifndef CYTHON_AVOID_BORROWED_REFS + #define CYTHON_AVOID_BORROWED_REFS 0 + #endif + #ifndef CYTHON_ASSUME_SAFE_MACROS + #define CYTHON_ASSUME_SAFE_MACROS 1 + #endif + #ifndef CYTHON_UNPACK_METHODS + #define CYTHON_UNPACK_METHODS 1 + #endif + #ifndef CYTHON_FAST_THREAD_STATE + #define CYTHON_FAST_THREAD_STATE 1 + #endif + #ifndef CYTHON_FAST_PYCALL + #define CYTHON_FAST_PYCALL 1 + #endif #endif -#if !defined(CYTHON_USE_PYLONG_INTERNALS) && CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 - #define CYTHON_USE_PYLONG_INTERNALS 1 +#if !defined(CYTHON_FAST_PYCCALL) +#define CYTHON_FAST_PYCCALL (CYTHON_FAST_PYCALL && PY_VERSION_HEX >= 0x030600B1) #endif #if CYTHON_USE_PYLONG_INTERNALS #include "longintrepr.h" @@ -79,23 +181,45 @@ #ifndef Py_TPFLAGS_HAVE_FINALIZE #define Py_TPFLAGS_HAVE_FINALIZE 0 #endif +#ifndef METH_FASTCALL + #define METH_FASTCALL 0x80 + typedef PyObject *(*__Pyx_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); +#else + #define __Pyx_PyCFunctionFast _PyCFunctionFast +#endif +#if CYTHON_FAST_PYCCALL +#define __Pyx_PyFastCFunction_Check(func)\ + ((PyCFunction_Check(func) && (METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))))) +#else +#define __Pyx_PyFastCFunction_Check(func) 0 +#endif #if PY_VERSION_HEX > 0x03030000 && defined(PyUnicode_KIND) #define CYTHON_PEP393_ENABLED 1 #define __Pyx_PyUnicode_READY(op) (likely(PyUnicode_IS_READY(op)) ?\ 0 : _PyUnicode_Ready((PyObject *)(op))) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_LENGTH(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) PyUnicode_READ_CHAR(u, i) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) PyUnicode_MAX_CHAR_VALUE(u) #define __Pyx_PyUnicode_KIND(u) PyUnicode_KIND(u) #define __Pyx_PyUnicode_DATA(u) PyUnicode_DATA(u) #define __Pyx_PyUnicode_READ(k, d, i) PyUnicode_READ(k, d, i) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) PyUnicode_WRITE(k, d, i, ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != (likely(PyUnicode_IS_READY(u)) ? PyUnicode_GET_LENGTH(u) : PyUnicode_GET_SIZE(u))) #else #define CYTHON_PEP393_ENABLED 0 + #define PyUnicode_1BYTE_KIND 1 + #define PyUnicode_2BYTE_KIND 2 + #define PyUnicode_4BYTE_KIND 4 #define __Pyx_PyUnicode_READY(op) (0) #define __Pyx_PyUnicode_GET_LENGTH(u) PyUnicode_GET_SIZE(u) #define __Pyx_PyUnicode_READ_CHAR(u, i) ((Py_UCS4)(PyUnicode_AS_UNICODE(u)[i])) + #define __Pyx_PyUnicode_MAX_CHAR_VALUE(u) ((sizeof(Py_UNICODE) == 2) ? 65535 : 1114111) #define __Pyx_PyUnicode_KIND(u) (sizeof(Py_UNICODE)) #define __Pyx_PyUnicode_DATA(u) ((void*)PyUnicode_AS_UNICODE(u)) #define __Pyx_PyUnicode_READ(k, d, i) ((void)(k), (Py_UCS4)(((Py_UNICODE*)d)[i])) + #define __Pyx_PyUnicode_WRITE(k, d, i, ch) (((void)(k)), ((Py_UNICODE*)d)[i] = ch) + #define __Pyx_PyUnicode_IS_TRUE(u) (0 != PyUnicode_GET_SIZE(u)) #endif #if CYTHON_COMPILING_IN_PYPY #define __Pyx_PyUnicode_Concat(a, b) PyNumber_Add(a, b) @@ -108,6 +232,24 @@ #if CYTHON_COMPILING_IN_PYPY && !defined(PyUnicode_Contains) #define PyUnicode_Contains(u, s) PySequence_Contains(u, s) #endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyByteArray_Check) + #define PyByteArray_Check(obj) PyObject_TypeCheck(obj, &PyByteArray_Type) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Format) + #define PyObject_Format(obj, fmt) PyObject_CallMethod(obj, "__format__", "O", fmt) +#endif +#if CYTHON_COMPILING_IN_PYPY && !defined(PyObject_Malloc) + #define PyObject_Malloc(s) PyMem_Malloc(s) + #define PyObject_Free(p) PyMem_Free(p) + #define PyObject_Realloc(p) PyMem_Realloc(p) +#endif +#if CYTHON_COMPILING_IN_PYSTON + #define __Pyx_PyCode_HasFreeVars(co) PyCode_HasFreeVars(co) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) PyFrame_SetLineNumber(frame, lineno) +#else + #define __Pyx_PyCode_HasFreeVars(co) (PyCode_GetNumFree(co) > 0) + #define __Pyx_PyFrame_SetLineNumber(frame, lineno) (frame)->f_lineno = (lineno) +#endif #define __Pyx_PyString_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : __Pyx_PyString_Format(a, b)) #define __Pyx_PyUnicode_FormatSafe(a, b) ((unlikely((a) == Py_None)) ? PyNumber_Remainder(a, b) : PyUnicode_Format(a, b)) #if PY_MAJOR_VERSION >= 3 @@ -115,6 +257,9 @@ #else #define __Pyx_PyString_Format(a, b) PyString_Format(a, b) #endif +#if PY_MAJOR_VERSION < 3 && !defined(PyObject_ASCII) + #define PyObject_ASCII(o) PyObject_Repr(o) +#endif #if PY_MAJOR_VERSION >= 3 #define PyBaseString_Type PyUnicode_Type #define PyStringObject PyUnicodeObject @@ -133,6 +278,7 @@ #define PySet_CheckExact(obj) (Py_TYPE(obj) == &PySet_Type) #endif #define __Pyx_TypeCheck(obj, type) PyObject_TypeCheck(obj, (PyTypeObject *)type) +#define __Pyx_PyException_Check(obj) __Pyx_TypeCheck(obj, PyExc_Exception) #if PY_MAJOR_VERSION >= 3 #define PyIntObject PyLongObject #define PyInt_Type PyLong_Type @@ -171,18 +317,20 @@ #else #define __Pyx_PyMethod_New(func, self, klass) PyMethod_New(func, self, klass) #endif -#if PY_VERSION_HEX >= 0x030500B1 -#define __Pyx_PyAsyncMethodsStruct PyAsyncMethods -#define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) -#elif CYTHON_COMPILING_IN_CPYTHON && PY_MAJOR_VERSION >= 3 -typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; -} __Pyx_PyAsyncMethodsStruct; -#define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) +#if CYTHON_USE_ASYNC_SLOTS + #if PY_VERSION_HEX >= 0x030500B1 + #define __Pyx_PyAsyncMethodsStruct PyAsyncMethods + #define __Pyx_PyType_AsAsync(obj) (Py_TYPE(obj)->tp_as_async) + #else + typedef struct { + unaryfunc am_await; + unaryfunc am_aiter; + unaryfunc am_anext; + } __Pyx_PyAsyncMethodsStruct; + #define __Pyx_PyType_AsAsync(obj) ((__Pyx_PyAsyncMethodsStruct*) (Py_TYPE(obj)->tp_reserved)) + #endif #else -#define __Pyx_PyType_AsAsync(obj) NULL + #define __Pyx_PyType_AsAsync(obj) NULL #endif #ifndef CYTHON_RESTRICT #if defined(__GNUC__) @@ -195,10 +343,39 @@ #define CYTHON_RESTRICT #endif #endif +#ifndef CYTHON_UNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) +# define CYTHON_UNUSED __attribute__ ((__unused__)) +# else +# define CYTHON_UNUSED +# endif +#endif +#ifndef CYTHON_MAYBE_UNUSED_VAR +# if defined(__cplusplus) + template void CYTHON_MAYBE_UNUSED_VAR( const T& ) { } +# else +# define CYTHON_MAYBE_UNUSED_VAR(x) (void)(x) +# endif +#endif +#ifndef CYTHON_NCP_UNUSED +# if CYTHON_COMPILING_IN_CPYTHON +# define CYTHON_NCP_UNUSED +# else +# define CYTHON_NCP_UNUSED CYTHON_UNUSED +# endif +#endif #define __Pyx_void_to_None(void_result) ((void)(void_result), Py_INCREF(Py_None), Py_None) #ifndef CYTHON_INLINE - #if defined(__GNUC__) + #if defined(__clang__) + #define CYTHON_INLINE __inline__ __attribute__ ((__unused__)) + #elif defined(__GNUC__) #define CYTHON_INLINE __inline__ #elif defined(_MSC_VER) #define CYTHON_INLINE __inline @@ -222,6 +399,11 @@ return value; } #endif +#if defined(__CYGWIN__) && defined(_LDBL_EQ_DBL) +#define __Pyx_truncl trunc +#else +#define __Pyx_truncl truncl +#endif #define __PYX_ERR(f_index, lineno, Ln_error) \ @@ -247,13 +429,15 @@ #define __PYX_HAVE__s3ql__deltadump #define __PYX_HAVE_API__s3ql__deltadump -#include "string.h" -#include "stdio.h" -#include "errno.h" -#include "stdlib.h" +#include +#include +#include +#include +#include +#include +#include #include "stdint.h" -#include "sys/types.h" -#include "unistd.h" +#include "stdio.h" #include "endian_indep.h" #include "sqlite3.h" #ifdef _OPENMP @@ -264,26 +448,6 @@ #define CYTHON_WITHOUT_ASSERTIONS #endif -#ifndef CYTHON_UNUSED -# if defined(__GNUC__) -# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -# elif defined(__ICC) || (defined(__INTEL_COMPILER) && !defined(_MSC_VER)) -# define CYTHON_UNUSED __attribute__ ((__unused__)) -# else -# define CYTHON_UNUSED -# endif -#endif -#ifndef CYTHON_NCP_UNUSED -# if CYTHON_COMPILING_IN_CPYTHON -# define CYTHON_NCP_UNUSED -# else -# define CYTHON_NCP_UNUSED CYTHON_UNUSED -# endif -#endif typedef struct {PyObject **p; const char *s; const Py_ssize_t n; const char* encoding; const char is_unicode; const char is_str; const char intern; } __Pyx_StringTabEntry; @@ -361,7 +525,7 @@ static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x); static CYTHON_INLINE Py_ssize_t __Pyx_PyIndex_AsSsize_t(PyObject*); static CYTHON_INLINE PyObject * __Pyx_PyInt_FromSize_t(size_t); -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_ASSUME_SAFE_MACROS #define __pyx_PyFloat_AsDouble(x) (PyFloat_CheckExact(x) ? PyFloat_AS_DOUBLE(x) : PyFloat_AsDouble(x)) #else #define __pyx_PyFloat_AsDouble(x) PyFloat_AsDouble(x) @@ -463,6 +627,7 @@ static PyObject *__pyx_b; static PyObject *__pyx_empty_tuple; static PyObject *__pyx_empty_bytes; +static PyObject *__pyx_empty_unicode; static int __pyx_lineno; static int __pyx_clineno = 0; static const char * __pyx_cfilenm= __FILE__; @@ -531,6 +696,7 @@ /* --- Runtime support code (head) --- */ +/* Refnanny.proto */ #ifndef CYTHON_REFNANNY #define CYTHON_REFNANNY 0 #endif @@ -593,7 +759,8 @@ #define __Pyx_CLEAR(r) do { PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);} while(0) #define __Pyx_XCLEAR(r) do { if((r) != NULL) {PyObject* tmp = ((PyObject*)(r)); r = NULL; __Pyx_DECREF(tmp);}} while(0) -#if CYTHON_COMPILING_IN_CPYTHON +/* PyObjectGetAttrStr.proto */ +#if CYTHON_USE_TYPE_SLOTS static CYTHON_INLINE PyObject* __Pyx_PyObject_GetAttrStr(PyObject* obj, PyObject* attr_name) { PyTypeObject* tp = Py_TYPE(obj); if (likely(tp->tp_getattro)) @@ -608,15 +775,36 @@ #define __Pyx_PyObject_GetAttrStr(o,n) PyObject_GetAttr(o,n) #endif +/* GetBuiltinName.proto */ static PyObject *__Pyx_GetBuiltinName(PyObject *name); +/* PyFunctionFastCall.proto */ +#if CYTHON_FAST_PYCALL +#define __Pyx_PyFunction_FastCall(func, args, nargs)\ + __Pyx_PyFunction_FastCallDict((func), (args), (nargs), NULL) +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs); +#else +#define __Pyx_PyFunction_FastCallDict(func, args, nargs, kwargs) _PyFunction_FastCallDict(func, args, nargs, kwargs) +#endif +#endif + +/* PyCFunctionFastCall.proto */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject *__Pyx_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs); +#else +#define __Pyx_PyCFunction_FastCall(func, args, nargs) (assert(0), NULL) +#endif + +/* PyObjectCall.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw); #else #define __Pyx_PyObject_Call(func, arg, kw) PyObject_Call(func, arg, kw) #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* PyThreadStateGet.proto */ +#if CYTHON_FAST_THREAD_STATE #define __Pyx_PyThreadState_declare PyThreadState *__pyx_tstate; #define __Pyx_PyThreadState_assign __pyx_tstate = PyThreadState_GET(); #else @@ -624,7 +812,8 @@ #define __Pyx_PyThreadState_assign #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* PyErrFetchRestore.proto */ +#if CYTHON_FAST_THREAD_STATE #define __Pyx_ErrRestoreWithState(type, value, tb) __Pyx_ErrRestoreInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrFetchWithState(type, value, tb) __Pyx_ErrFetchInState(PyThreadState_GET(), type, value, tb) #define __Pyx_ErrRestore(type, value, tb) __Pyx_ErrRestoreInState(__pyx_tstate, type, value, tb) @@ -638,16 +827,21 @@ #define __Pyx_ErrFetch(type, value, tb) PyErr_Fetch(type, value, tb) #endif +/* RaiseException.proto */ static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, PyObject *cause); +/* GetModuleGlobalName.proto */ static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name); +/* PyObjectCallMethO.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg); #endif +/* PyObjectCallOneArg.proto */ static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg); +/* GetItemInt.proto */ #define __Pyx_GetItemInt(o, i, type, is_signed, to_py_func, is_list, wraparound, boundscheck)\ (__Pyx_fits_Py_ssize_t(i, type, is_signed) ?\ __Pyx_GetItemInt_Fast(o, (Py_ssize_t)i, is_list, wraparound, boundscheck) :\ @@ -669,27 +863,35 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, int wraparound, int boundscheck); +/* PyObjectCallNoArg.proto */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func); #else #define __Pyx_PyObject_CallNoArg(func) __Pyx_PyObject_Call(func, __pyx_empty_tuple, NULL) #endif +/* RaiseArgTupleInvalid.proto */ static void __Pyx_RaiseArgtupleInvalid(const char* func_name, int exact, Py_ssize_t num_min, Py_ssize_t num_max, Py_ssize_t num_found); +/* RaiseDoubleKeywords.proto */ static void __Pyx_RaiseDoubleKeywordsError(const char* func_name, PyObject* kw_name); +/* ParseKeywords.proto */ static int __Pyx_ParseOptionalKeywords(PyObject *kwds, PyObject **argnames[],\ PyObject *kwds2, PyObject *values[], Py_ssize_t num_pos_args,\ const char* function_name); +/* IncludeStringH.proto */ #include +/* BytesEquals.proto */ static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals); +/* UnicodeEquals.proto */ static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals); +/* PyObjectLookupSpecial.proto */ #if CYTHON_COMPILING_IN_CPYTHON && PY_VERSION_HEX >= 0x02070000 static CYTHON_INLINE PyObject* __Pyx_PyObject_LookupSpecial(PyObject* obj, PyObject* attr_name) { PyObject *res; @@ -715,8 +917,10 @@ #define __Pyx_PyObject_LookupSpecial(o,n) __Pyx_PyObject_GetAttrStr(o,n) #endif +/* FetchCommonType.proto */ static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type); +/* CythonFunction.proto */ #define __Pyx_CyFunction_USED 1 #include #define __Pyx_CYFUNCTION_STATICMETHOD 0x01 @@ -770,7 +974,8 @@ PyObject *dict); static int __pyx_CyFunction_init(void); -#if CYTHON_COMPILING_IN_CPYTHON +/* ListCompAppend.proto */ +#if CYTHON_USE_PYLIST_INTERNALS && CYTHON_ASSUME_SAFE_MACROS static CYTHON_INLINE int __Pyx_ListComp_Append(PyObject* list, PyObject* x) { PyListObject* L = (PyListObject*) list; Py_ssize_t len = Py_SIZE(list); @@ -786,7 +991,8 @@ #define __Pyx_ListComp_Append(L,x) PyList_Append(L,x) #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* SaveResetException.proto */ +#if CYTHON_FAST_THREAD_STATE #define __Pyx_ExceptionSave(type, value, tb) __Pyx__ExceptionSave(__pyx_tstate, type, value, tb) static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); #define __Pyx_ExceptionReset(type, value, tb) __Pyx__ExceptionReset(__pyx_tstate, type, value, tb) @@ -796,34 +1002,40 @@ #define __Pyx_ExceptionReset(type, value, tb) PyErr_SetExcInfo(type, value, tb) #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* GetException.proto */ +#if CYTHON_FAST_THREAD_STATE #define __Pyx_GetException(type, value, tb) __Pyx__GetException(__pyx_tstate, type, value, tb) static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); #else static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb); #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* SwapException.proto */ +#if CYTHON_FAST_THREAD_STATE #define __Pyx_ExceptionSwap(type, value, tb) __Pyx__ExceptionSwap(__pyx_tstate, type, value, tb) static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb); #else static CYTHON_INLINE void __Pyx_ExceptionSwap(PyObject **type, PyObject **value, PyObject **tb); #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* PyIntBinop.proto */ +#if !CYTHON_COMPILING_IN_PYPY static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, long intval, int inplace); #else #define __Pyx_PyInt_EqObjC(op1, op2, intval, inplace)\ PyObject_RichCompare(op1, op2, Py_EQ) #endif +/* Import.proto */ static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level); +/* ImportFrom.proto */ static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name); +/* CodeObjectCache.proto */ typedef struct { - int code_line; PyCodeObject* code_object; + int code_line; } __Pyx_CodeObjectCacheEntry; struct __Pyx_CodeObjectCache { int count; @@ -835,25 +1047,35 @@ static PyCodeObject *__pyx_find_code_object(int code_line); static void __pyx_insert_code_object(int code_line, PyCodeObject* code_object); +/* AddTraceback.proto */ static void __Pyx_AddTraceback(const char *funcname, int c_line, int py_line, const char *filename); +/* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value); +/* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value); +/* CIntToPy.proto */ static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int64_t(int64_t value); +/* CIntFromPy.proto */ static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *); +/* CIntFromPy.proto */ static CYTHON_INLINE int64_t __Pyx_PyInt_As_int64_t(PyObject *); +/* CIntFromPy.proto */ static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *); +/* CIntFromPy.proto */ static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *); +/* CheckBinaryVersion.proto */ static int __Pyx_check_binary_version(void); +/* PyIdentifierFromString.proto */ #if !defined(__Pyx_PyIdentifier_FromString) #if PY_MAJOR_VERSION < 3 #define __Pyx_PyIdentifier_FromString(s) PyString_FromString(s) @@ -862,10 +1084,13 @@ #endif #endif +/* ModuleImport.proto */ static PyObject *__Pyx_ImportModule(const char *name); +/* TypeImport.proto */ static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict); +/* InitStrings.proto */ static int __Pyx_InitStrings(__Pyx_StringTabEntry *t); @@ -1388,7 +1613,7 @@ PyObject *__pyx_t_3 = NULL; PyObject *__pyx_t_4 = NULL; PyObject *__pyx_t_5 = NULL; - Py_ssize_t __pyx_t_6; + int __pyx_t_6; PyObject *__pyx_t_7 = NULL; __Pyx_RefNannySetupContext("raise_from_errno", 0); if (__pyx_optional_args) { @@ -1411,7 +1636,7 @@ __Pyx_INCREF(__pyx_v_err_class); __pyx_t_4 = __pyx_v_err_class; __pyx_t_5 = NULL; __pyx_t_6 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_4))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_5)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); @@ -1421,20 +1646,42 @@ __pyx_t_6 = 1; } } - __pyx_t_7 = PyTuple_New(2+__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 139, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_7); - if (__pyx_t_5) { - __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[3] = {__pyx_t_5, __pyx_t_2, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_6, 2+__pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[3] = {__pyx_t_5, __pyx_t_2, __pyx_t_3}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-__pyx_t_6, 2+__pyx_t_6); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_7 = PyTuple_New(2+__pyx_t_6); if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_7); + if (__pyx_t_5) { + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_7, 0, __pyx_t_5); __pyx_t_5 = NULL; + } + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_6, __pyx_t_2); + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_6, __pyx_t_3); + __pyx_t_2 = 0; + __pyx_t_3 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; } - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_7, 0+__pyx_t_6, __pyx_t_2); - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_7, 1+__pyx_t_6, __pyx_t_3); - __pyx_t_2 = 0; - __pyx_t_3 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_7, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 139, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_7); __pyx_t_7 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_Raise(__pyx_t_1, 0, 0, 0); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -1751,7 +1998,7 @@ __pyx_t_3 = __Pyx_PyInt_From_int(__pyx_v_rc); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 185, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_5 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_4))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_4))) { __pyx_t_5 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_5)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); @@ -1765,15 +2012,35 @@ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_2); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 185, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_t_3}; + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_5, __pyx_t_3}; + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_5); __pyx_t_5 = 0; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_5); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_5); __pyx_t_5 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 185, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __pyx_v_exc = __pyx_t_2; @@ -1790,7 +2057,7 @@ __Pyx_GOTREF(__pyx_t_4); __Pyx_INCREF(((PyObject *)Py_TYPE(__pyx_v_exc))); __pyx_t_6 = ((PyObject *)Py_TYPE(__pyx_v_exc)); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); @@ -1804,15 +2071,35 @@ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_2); } else { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 186, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 186, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_2); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_4}; + __pyx_t_2 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_2 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_5, NULL); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 186, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_2); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_Raise(__pyx_t_2, 0, 0, 0); @@ -1949,7 +2236,7 @@ if (likely(!__pyx_t_5)) { if (likely(PyList_CheckExact(__pyx_t_4))) { if (__pyx_t_1 >= PyList_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_3 = PyList_GET_ITEM(__pyx_t_4, __pyx_t_1); __Pyx_INCREF(__pyx_t_3); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(0, 206, __pyx_L1_error) #else __pyx_t_3 = PySequence_ITEM(__pyx_t_4, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 206, __pyx_L1_error) @@ -1957,7 +2244,7 @@ #endif } else { if (__pyx_t_1 >= PyTuple_GET_SIZE(__pyx_t_4)) break; - #if CYTHON_COMPILING_IN_CPYTHON + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_1); __Pyx_INCREF(__pyx_t_3); __pyx_t_1++; if (unlikely(0 < 0)) __PYX_ERR(0, 206, __pyx_L1_error) #else __pyx_t_3 = PySequence_ITEM(__pyx_t_4, __pyx_t_1); __pyx_t_1++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 206, __pyx_L1_error) @@ -2216,7 +2503,7 @@ __pyx_t_2 = __Pyx_PyObject_GetAttrStr(__pyx_v_fh, __pyx_n_s_fileno); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 227, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_2))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_2))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); @@ -2367,11 +2654,11 @@ PyObject *__pyx_t_1 = NULL; PyObject *__pyx_t_2 = NULL; PyObject *__pyx_t_3 = NULL; - Py_ssize_t __pyx_t_4; + int __pyx_t_4; PyObject *__pyx_t_5 = NULL; int __pyx_t_6; - PyObject *(*__pyx_t_7)(PyObject *); - int __pyx_t_8; + Py_ssize_t __pyx_t_7; + PyObject *(*__pyx_t_8)(PyObject *); int __pyx_t_9; __Pyx_RefNannySetupContext("check_sqlite", 0); @@ -2388,7 +2675,7 @@ __Pyx_GOTREF(__pyx_t_3); __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __pyx_t_2 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_2)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); @@ -2442,7 +2729,7 @@ */ __pyx_t_3 = NULL; __pyx_t_4 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_2))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_2))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_2); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_2); @@ -2452,23 +2739,41 @@ __pyx_t_4 = 1; } } - __pyx_t_5 = PyTuple_New(3+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 253, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - if (__pyx_t_3) { - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[4] = {__pyx_t_3, __pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit, __pyx_v_apsw_sqlite_version, __pyx_v_s3ql_sqlite_version}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 3+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_2)) { + PyObject *__pyx_temp[4] = {__pyx_t_3, __pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit, __pyx_v_apsw_sqlite_version, __pyx_v_s3ql_sqlite_version}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_2, __pyx_temp+1-__pyx_t_4, 3+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_5 = PyTuple_New(3+__pyx_t_4); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + if (__pyx_t_3) { + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_3); __pyx_t_3 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); + __Pyx_GIVEREF(__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); + PyTuple_SET_ITEM(__pyx_t_5, 0+__pyx_t_4, __pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); + __Pyx_INCREF(__pyx_v_apsw_sqlite_version); + __Pyx_GIVEREF(__pyx_v_apsw_sqlite_version); + PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_apsw_sqlite_version); + __Pyx_INCREF(__pyx_v_s3ql_sqlite_version); + __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_version); + PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_v_s3ql_sqlite_version); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 253, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; } - __Pyx_INCREF(__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); - __Pyx_GIVEREF(__pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); - PyTuple_SET_ITEM(__pyx_t_5, 0+__pyx_t_4, __pyx_kp_u_apsw_sqlite_version_s_s3ql_sqlit); - __Pyx_INCREF(__pyx_v_apsw_sqlite_version); - __Pyx_GIVEREF(__pyx_v_apsw_sqlite_version); - PyTuple_SET_ITEM(__pyx_t_5, 1+__pyx_t_4, __pyx_v_apsw_sqlite_version); - __Pyx_INCREF(__pyx_v_s3ql_sqlite_version); - __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_version); - PyTuple_SET_ITEM(__pyx_t_5, 2+__pyx_t_4, __pyx_v_s3ql_sqlite_version); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_2, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 253, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -2577,35 +2882,35 @@ __Pyx_GOTREF(__pyx_t_2); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; if (likely(PyList_CheckExact(__pyx_t_2)) || PyTuple_CheckExact(__pyx_t_2)) { - __pyx_t_1 = __pyx_t_2; __Pyx_INCREF(__pyx_t_1); __pyx_t_4 = 0; - __pyx_t_7 = NULL; + __pyx_t_1 = __pyx_t_2; __Pyx_INCREF(__pyx_t_1); __pyx_t_7 = 0; + __pyx_t_8 = NULL; } else { - __pyx_t_4 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 263, __pyx_L1_error) + __pyx_t_7 = -1; __pyx_t_1 = PyObject_GetIter(__pyx_t_2); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 263, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_1); - __pyx_t_7 = Py_TYPE(__pyx_t_1)->tp_iternext; if (unlikely(!__pyx_t_7)) __PYX_ERR(0, 263, __pyx_L1_error) + __pyx_t_8 = Py_TYPE(__pyx_t_1)->tp_iternext; if (unlikely(!__pyx_t_8)) __PYX_ERR(0, 263, __pyx_L1_error) } __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; for (;;) { - if (likely(!__pyx_t_7)) { + if (likely(!__pyx_t_8)) { if (likely(PyList_CheckExact(__pyx_t_1))) { - if (__pyx_t_4 >= PyList_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_4); __Pyx_INCREF(__pyx_t_2); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 263, __pyx_L1_error) + if (__pyx_t_7 >= PyList_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_2 = PyList_GET_ITEM(__pyx_t_1, __pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 263, __pyx_L1_error) #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) + __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif } else { - if (__pyx_t_4 >= PyTuple_GET_SIZE(__pyx_t_1)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_4); __Pyx_INCREF(__pyx_t_2); __pyx_t_4++; if (unlikely(0 < 0)) __PYX_ERR(0, 263, __pyx_L1_error) + if (__pyx_t_7 >= PyTuple_GET_SIZE(__pyx_t_1)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_2 = PyTuple_GET_ITEM(__pyx_t_1, __pyx_t_7); __Pyx_INCREF(__pyx_t_2); __pyx_t_7++; if (unlikely(0 < 0)) __PYX_ERR(0, 263, __pyx_L1_error) #else - __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_4); __pyx_t_4++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) + __pyx_t_2 = PySequence_ITEM(__pyx_t_1, __pyx_t_7); __pyx_t_7++; if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 263, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); #endif } } else { - __pyx_t_2 = __pyx_t_7(__pyx_t_1); + __pyx_t_2 = __pyx_t_8(__pyx_t_1); if (unlikely(!__pyx_t_2)) { PyObject* exc_type = PyErr_Occurred(); if (exc_type) { @@ -2626,8 +2931,8 @@ * if buf is NULL: * break */ - __pyx_t_8 = __Pyx_PyInt_As_int(__pyx_v_idx); if (unlikely((__pyx_t_8 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 264, __pyx_L1_error) - __pyx_v_buf = sqlite3_compileoption_get(__pyx_t_8); + __pyx_t_4 = __Pyx_PyInt_As_int(__pyx_v_idx); if (unlikely((__pyx_t_4 == (int)-1) && PyErr_Occurred())) __PYX_ERR(0, 264, __pyx_L1_error) + __pyx_v_buf = sqlite3_compileoption_get(__pyx_t_4); /* "s3ql/deltadump.pyx":265 * for idx in itertools.count(0): @@ -2702,7 +3007,7 @@ */ __pyx_t_2 = NULL; __pyx_t_4 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_5))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_5))) { __pyx_t_2 = PyMethod_GET_SELF(__pyx_t_5); if (likely(__pyx_t_2)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_5); @@ -2712,23 +3017,41 @@ __pyx_t_4 = 1; } } - __pyx_t_3 = PyTuple_New(3+__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 269, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_3); - if (__pyx_t_2) { - __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __pyx_t_2 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_5)) { + PyObject *__pyx_temp[4] = {__pyx_t_2, __pyx_kp_u_apsw_sqlite_compile_options_s_s3, __pyx_v_apsw_sqlite_options, __pyx_v_s3ql_sqlite_options}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_4, 3+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_5)) { + PyObject *__pyx_temp[4] = {__pyx_t_2, __pyx_kp_u_apsw_sqlite_compile_options_s_s3, __pyx_v_apsw_sqlite_options, __pyx_v_s3ql_sqlite_options}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_5, __pyx_temp+1-__pyx_t_4, 3+__pyx_t_4); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_2); __pyx_t_2 = 0; + __Pyx_GOTREF(__pyx_t_1); + } else + #endif + { + __pyx_t_3 = PyTuple_New(3+__pyx_t_4); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_3); + if (__pyx_t_2) { + __Pyx_GIVEREF(__pyx_t_2); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_2); __pyx_t_2 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_apsw_sqlite_compile_options_s_s3); + __Pyx_GIVEREF(__pyx_kp_u_apsw_sqlite_compile_options_s_s3); + PyTuple_SET_ITEM(__pyx_t_3, 0+__pyx_t_4, __pyx_kp_u_apsw_sqlite_compile_options_s_s3); + __Pyx_INCREF(__pyx_v_apsw_sqlite_options); + __Pyx_GIVEREF(__pyx_v_apsw_sqlite_options); + PyTuple_SET_ITEM(__pyx_t_3, 1+__pyx_t_4, __pyx_v_apsw_sqlite_options); + __Pyx_INCREF(__pyx_v_s3ql_sqlite_options); + __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_options); + PyTuple_SET_ITEM(__pyx_t_3, 2+__pyx_t_4, __pyx_v_s3ql_sqlite_options); + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 269, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } - __Pyx_INCREF(__pyx_kp_u_apsw_sqlite_compile_options_s_s3); - __Pyx_GIVEREF(__pyx_kp_u_apsw_sqlite_compile_options_s_s3); - PyTuple_SET_ITEM(__pyx_t_3, 0+__pyx_t_4, __pyx_kp_u_apsw_sqlite_compile_options_s_s3); - __Pyx_INCREF(__pyx_v_apsw_sqlite_options); - __Pyx_GIVEREF(__pyx_v_apsw_sqlite_options); - PyTuple_SET_ITEM(__pyx_t_3, 1+__pyx_t_4, __pyx_v_apsw_sqlite_options); - __Pyx_INCREF(__pyx_v_s3ql_sqlite_options); - __Pyx_GIVEREF(__pyx_v_s3ql_sqlite_options); - PyTuple_SET_ITEM(__pyx_t_3, 2+__pyx_t_4, __pyx_v_s3ql_sqlite_options); - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_5, __pyx_t_3, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 269, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -3308,11 +3631,11 @@ PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; PyObject *__pyx_t_9 = NULL; - Py_ssize_t __pyx_t_10; + int __pyx_t_10; PyObject *__pyx_t_11 = NULL; const char *__pyx_t_12; - int __pyx_t_13; - FILE *__pyx_t_14; + FILE *__pyx_t_13; + Py_ssize_t __pyx_t_14; void *__pyx_t_15; PyObject *(*__pyx_t_16)(PyObject *); char *__pyx_t_17; @@ -3324,10 +3647,12 @@ __Pyx_RefNannySetupContext("dump_table", 0); __pyx_cur_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *)__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct__dump_table(__pyx_ptype_4s3ql_9deltadump___pyx_scope_struct__dump_table, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { - __Pyx_RefNannyFinishContext(); - return NULL; + __pyx_cur_scope = ((struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct__dump_table *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 280, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_GOTREF(__pyx_cur_scope); /* "s3ql/deltadump.pyx":318 * cdef int64_t row_count @@ -3375,7 +3700,7 @@ __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ExitStack); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 321, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_4)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); @@ -3397,7 +3722,7 @@ __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 321, __pyx_L4_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); @@ -3445,7 +3770,7 @@ __Pyx_GOTREF(__pyx_t_1); __pyx_t_6 = NULL; __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); @@ -3455,20 +3780,40 @@ __pyx_t_10 = 1; } } - __pyx_t_11 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 323, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_11); - if (__pyx_t_6) { - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6); __pyx_t_6 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_kp_u_Opening_connection_to_s, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_kp_u_Opening_connection_to_s, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_11 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_kp_u_Opening_connection_to_s); + PyTuple_SET_ITEM(__pyx_t_11, 0+__pyx_t_10, __pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_11, 1+__pyx_t_10, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 323, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } - __Pyx_INCREF(__pyx_kp_u_Opening_connection_to_s); - __Pyx_GIVEREF(__pyx_kp_u_Opening_connection_to_s); - PyTuple_SET_ITEM(__pyx_t_11, 0+__pyx_t_10, __pyx_kp_u_Opening_connection_to_s); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_11, 1+__pyx_t_10, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 323, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3490,7 +3835,7 @@ __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); @@ -3509,7 +3854,7 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = NULL; __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -3519,20 +3864,40 @@ __pyx_t_10 = 1; } } - __pyx_t_1 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 324, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - if (__pyx_t_6) { - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_3, __pyx_n_u_surrogateescape}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_3, __pyx_n_u_surrogateescape}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_1 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+__pyx_t_10, __pyx_t_3); + __Pyx_INCREF(__pyx_n_u_surrogateescape); + __Pyx_GIVEREF(__pyx_n_u_surrogateescape); + PyTuple_SET_ITEM(__pyx_t_1, 1+__pyx_t_10, __pyx_n_u_surrogateescape); + __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 324, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_1, 0+__pyx_t_10, __pyx_t_3); - __Pyx_INCREF(__pyx_n_u_surrogateescape); - __Pyx_GIVEREF(__pyx_n_u_surrogateescape); - PyTuple_SET_ITEM(__pyx_t_1, 1+__pyx_t_10, __pyx_n_u_surrogateescape); - __pyx_t_3 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 324, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __pyx_v_dbfile_b = __pyx_t_4; __pyx_t_4 = 0; @@ -3553,7 +3918,7 @@ * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), * SQLITE_OK, sqlite3_db)) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_open_v2(__pyx_t_12, (&__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OPEN_READONLY, NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 325, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_open_v2(__pyx_t_12, (&__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OPEN_READONLY, NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 325, __pyx_L8_error) /* "s3ql/deltadump.pyx":328 * SQLITE_OPEN_READONLY, NULL), @@ -3567,7 +3932,7 @@ __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_lambda, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 328, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -3581,15 +3946,35 @@ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 328, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 328, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 328, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3601,7 +3986,7 @@ * SQLITE_OK, sqlite3_db) * */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_extended_result_codes(__pyx_cur_scope->__pyx_v_sqlite3_db, 1), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 330, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_extended_result_codes(__pyx_cur_scope->__pyx_v_sqlite3_db, 1), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 330, __pyx_L8_error) /* "s3ql/deltadump.pyx":334 * @@ -3610,8 +3995,8 @@ * cm.callback(lambda: fclose(fp)) * */ - __pyx_t_14 = __pyx_f_4s3ql_9deltadump_dup_to_fp(__pyx_v_fh, ((const char *)"wb")); if (unlikely(__pyx_t_14 == NULL)) __PYX_ERR(0, 334, __pyx_L8_error) - __pyx_cur_scope->__pyx_v_fp = __pyx_t_14; + __pyx_t_13 = __pyx_f_4s3ql_9deltadump_dup_to_fp(__pyx_v_fh, ((const char *)"wb")); if (unlikely(__pyx_t_13 == NULL)) __PYX_ERR(0, 334, __pyx_L8_error) + __pyx_cur_scope->__pyx_v_fp = __pyx_t_13; /* "s3ql/deltadump.pyx":335 * # Get FILE* for buffered reading from *fh* @@ -3625,7 +4010,7 @@ __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_1lambda1, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 335, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_1 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -3639,15 +4024,35 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 335, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 335, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3659,8 +4064,8 @@ * cm.callback(lambda: free(col_args)) * cm.callback(lambda: free(col_types)) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_prep_columns(__pyx_v_columns, (&__pyx_cur_scope->__pyx_v_col_types), (&__pyx_cur_scope->__pyx_v_col_args)); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 338, __pyx_L8_error) - __pyx_v_col_count = __pyx_t_13; + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_prep_columns(__pyx_v_columns, (&__pyx_cur_scope->__pyx_v_col_types), (&__pyx_cur_scope->__pyx_v_col_args)); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 338, __pyx_L8_error) + __pyx_v_col_count = __pyx_t_10; /* "s3ql/deltadump.pyx":339 * # Allocate col_args and col_types @@ -3674,7 +4079,7 @@ __pyx_t_3 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_2lambda2, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 339, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -3688,15 +4093,35 @@ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 339, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 339, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 339, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3713,7 +4138,7 @@ __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_3lambda3, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 340, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -3727,15 +4152,35 @@ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 340, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 340, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 340, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3747,8 +4192,8 @@ * cm.callback(lambda: free(int64_prev)) * */ - __pyx_t_10 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 343, __pyx_L8_error) - __pyx_t_15 = __pyx_f_4s3ql_9deltadump_calloc(((size_t)__pyx_t_10), (sizeof(int64_t))); if (unlikely(__pyx_t_15 == NULL)) __PYX_ERR(0, 343, __pyx_L8_error) + __pyx_t_14 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_14 == -1)) __PYX_ERR(0, 343, __pyx_L8_error) + __pyx_t_15 = __pyx_f_4s3ql_9deltadump_calloc(((size_t)__pyx_t_14), (sizeof(int64_t))); if (unlikely(__pyx_t_15 == NULL)) __PYX_ERR(0, 343, __pyx_L8_error) __pyx_cur_scope->__pyx_v_int64_prev = ((int64_t *)__pyx_t_15); /* "s3ql/deltadump.pyx":344 @@ -3763,7 +4208,7 @@ __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_4lambda4, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 344, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_1 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -3777,15 +4222,35 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 344, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_3); - __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 344, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_3 = PyTuple_New(1+1); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_3, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_3, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 344, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -3802,29 +4267,29 @@ __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 347, __pyx_L18_error) __Pyx_GOTREF(__pyx_t_4); if (likely(PyList_CheckExact(__pyx_v_columns)) || PyTuple_CheckExact(__pyx_v_columns)) { - __pyx_t_11 = __pyx_v_columns; __Pyx_INCREF(__pyx_t_11); __pyx_t_10 = 0; + __pyx_t_11 = __pyx_v_columns; __Pyx_INCREF(__pyx_t_11); __pyx_t_14 = 0; __pyx_t_16 = NULL; } else { - __pyx_t_10 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_columns); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 347, __pyx_L18_error) + __pyx_t_14 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_columns); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 347, __pyx_L18_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_16 = Py_TYPE(__pyx_t_11)->tp_iternext; if (unlikely(!__pyx_t_16)) __PYX_ERR(0, 347, __pyx_L18_error) } for (;;) { if (likely(!__pyx_t_16)) { if (likely(PyList_CheckExact(__pyx_t_11))) { - if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_11)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_3 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_3); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 347, __pyx_L18_error) + if (__pyx_t_14 >= PyList_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_14); __Pyx_INCREF(__pyx_t_3); __pyx_t_14++; if (unlikely(0 < 0)) __PYX_ERR(0, 347, __pyx_L18_error) #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) + __pyx_t_3 = PySequence_ITEM(__pyx_t_11, __pyx_t_14); __pyx_t_14++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) __Pyx_GOTREF(__pyx_t_3); #endif } else { - if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_11)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_3); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 347, __pyx_L18_error) + if (__pyx_t_14 >= PyTuple_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_3 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_14); __Pyx_INCREF(__pyx_t_3); __pyx_t_14++; if (unlikely(0 < 0)) __PYX_ERR(0, 347, __pyx_L18_error) #else - __pyx_t_3 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) + __pyx_t_3 = PySequence_ITEM(__pyx_t_11, __pyx_t_14); __pyx_t_14++; if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 347, __pyx_L18_error) __Pyx_GOTREF(__pyx_t_3); #endif } @@ -3919,7 +4384,7 @@ * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_finalize(stmt), * SQLITE_OK, sqlite3_db)) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_17, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 350, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_17, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 350, __pyx_L8_error) /* "s3ql/deltadump.pyx":352 * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, query, -1, &stmt, NULL), @@ -3933,7 +4398,7 @@ __pyx_t_3 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10dump_table_5lambda5, 0, __pyx_n_s_dump_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 352, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); @@ -3947,15 +4412,35 @@ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_GOTREF(__pyx_t_11); } else { - __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 352, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_3); - __pyx_t_3 = 0; - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_1, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 352, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_11 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_3}; + __pyx_t_11 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_3); + __pyx_t_3 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_1, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 352, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; @@ -3972,7 +4457,7 @@ __pyx_t_1 = PyUnicode_Format(__pyx_kp_u_SELECT_COUNT_rowid_FROM_s, __pyx_v_table); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 355, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); @@ -3986,18 +4471,38 @@ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_11); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 355, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 355, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_11 = __Pyx_PyFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_4)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_11 = __Pyx_PyCFunction_FastCall(__pyx_t_4, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_4, __pyx_t_6, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 355, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; - __pyx_t_18 = __Pyx_PyInt_As_int64_t(__pyx_t_11); if (unlikely((__pyx_t_18 == (int64_t)-1) && PyErr_Occurred())) __PYX_ERR(0, 355, __pyx_L8_error) + __pyx_t_18 = __Pyx_PyInt_As_int64_t(__pyx_t_11); if (unlikely((__pyx_t_18 == ((int64_t)-1)) && PyErr_Occurred())) __PYX_ERR(0, 355, __pyx_L8_error) __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __pyx_v_row_count = __pyx_t_18; @@ -4017,7 +4522,7 @@ __Pyx_GOTREF(__pyx_t_4); __pyx_t_1 = NULL; __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); @@ -4027,23 +4532,43 @@ __pyx_t_10 = 1; } } - __pyx_t_3 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 356, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_3); - if (__pyx_t_1) { - __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[4] = {__pyx_t_1, __pyx_kp_u_dump_table_s_writing_d_rows, __pyx_v_table, __pyx_t_4}; + __pyx_t_11 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_10, 3+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[4] = {__pyx_t_1, __pyx_kp_u_dump_table_s_writing_d_rows, __pyx_v_table, __pyx_t_4}; + __pyx_t_11 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-__pyx_t_10, 3+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + { + __pyx_t_3 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_3); + if (__pyx_t_1) { + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_t_1); __pyx_t_1 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_dump_table_s_writing_d_rows); + __Pyx_GIVEREF(__pyx_kp_u_dump_table_s_writing_d_rows); + PyTuple_SET_ITEM(__pyx_t_3, 0+__pyx_t_10, __pyx_kp_u_dump_table_s_writing_d_rows); + __Pyx_INCREF(__pyx_v_table); + __Pyx_GIVEREF(__pyx_v_table); + PyTuple_SET_ITEM(__pyx_t_3, 1+__pyx_t_10, __pyx_v_table); + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_3, 2+__pyx_t_10, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_3, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 356, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; } - __Pyx_INCREF(__pyx_kp_u_dump_table_s_writing_d_rows); - __Pyx_GIVEREF(__pyx_kp_u_dump_table_s_writing_d_rows); - PyTuple_SET_ITEM(__pyx_t_3, 0+__pyx_t_10, __pyx_kp_u_dump_table_s_writing_d_rows); - __Pyx_INCREF(__pyx_v_table); - __Pyx_GIVEREF(__pyx_v_table); - PyTuple_SET_ITEM(__pyx_t_3, 1+__pyx_t_10, __pyx_v_table); - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_3, 2+__pyx_t_10, __pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_3, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 356, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; @@ -4054,7 +4579,7 @@ * * # Iterate through rows */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_write_integer(__pyx_v_row_count, __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 357, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_write_integer(__pyx_v_row_count, __pyx_cur_scope->__pyx_v_fp); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 357, __pyx_L8_error) /* "s3ql/deltadump.pyx":360 * @@ -4109,7 +4634,7 @@ * * for i in range(col_count): */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(__pyx_v_rc, SQLITE_ROW, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 364, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(__pyx_v_rc, SQLITE_ROW, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 364, __pyx_L8_error) /* "s3ql/deltadump.pyx":366 * SQLITE_CHECK_RC(rc, SQLITE_ROW, sqlite3_db) @@ -4118,8 +4643,8 @@ * if sqlite3_column_type(stmt, i) is SQLITE_NULL: * raise ValueError("Can't dump NULL values") */ - __pyx_t_13 = __pyx_v_col_count; - for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_13; __pyx_t_19+=1) { + __pyx_t_10 = __pyx_v_col_count; + for (__pyx_t_19 = 0; __pyx_t_19 < __pyx_t_10; __pyx_t_19+=1) { __pyx_v_i = __pyx_t_19; /* "s3ql/deltadump.pyx":367 @@ -5354,10 +5879,10 @@ PyObject *__pyx_t_7 = NULL; PyObject *__pyx_t_8 = NULL; PyObject *__pyx_t_9 = NULL; - Py_ssize_t __pyx_t_10; + int __pyx_t_10; PyObject *__pyx_t_11 = NULL; const char *__pyx_t_12; - int __pyx_t_13; + Py_ssize_t __pyx_t_13; PyObject *__pyx_t_14 = NULL; char *__pyx_t_15; int __pyx_t_16; @@ -5379,10 +5904,12 @@ __Pyx_RefNannySetupContext("load_table", 0); __pyx_cur_scope = (struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *)__pyx_tp_new_4s3ql_9deltadump___pyx_scope_struct_1_load_table(__pyx_ptype_4s3ql_9deltadump___pyx_scope_struct_1_load_table, __pyx_empty_tuple, NULL); if (unlikely(!__pyx_cur_scope)) { - __Pyx_RefNannyFinishContext(); - return NULL; + __pyx_cur_scope = ((struct __pyx_obj_4s3ql_9deltadump___pyx_scope_struct_1_load_table *)Py_None); + __Pyx_INCREF(Py_None); + __PYX_ERR(0, 401, __pyx_L1_error) + } else { + __Pyx_GOTREF(__pyx_cur_scope); } - __Pyx_GOTREF(__pyx_cur_scope); /* "s3ql/deltadump.pyx":431 * cdef int64_t row_count, int64, tmp @@ -5430,7 +5957,7 @@ __pyx_t_3 = __Pyx_GetModuleGlobalName(__pyx_n_s_ExitStack); if (unlikely(!__pyx_t_3)) __PYX_ERR(0, 434, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_3); __pyx_t_4 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_4)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); @@ -5452,7 +5979,7 @@ __pyx_t_4 = __Pyx_PyObject_LookupSpecial(__pyx_t_1, __pyx_n_s_enter); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 434, __pyx_L4_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_4))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_4))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_4); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_4); @@ -5500,7 +6027,7 @@ __Pyx_GOTREF(__pyx_t_1); __pyx_t_6 = NULL; __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); @@ -5510,20 +6037,40 @@ __pyx_t_10 = 1; } } - __pyx_t_11 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 436, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_11); - if (__pyx_t_6) { - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6); __pyx_t_6 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_kp_u_Opening_connection_to_s, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_kp_u_Opening_connection_to_s, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_11 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_11, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_kp_u_Opening_connection_to_s); + PyTuple_SET_ITEM(__pyx_t_11, 0+__pyx_t_10, __pyx_kp_u_Opening_connection_to_s); + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_11, 1+__pyx_t_10, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 436, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; } - __Pyx_INCREF(__pyx_kp_u_Opening_connection_to_s); - __Pyx_GIVEREF(__pyx_kp_u_Opening_connection_to_s); - PyTuple_SET_ITEM(__pyx_t_11, 0+__pyx_t_10, __pyx_kp_u_Opening_connection_to_s); - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_11, 1+__pyx_t_10, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_11, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 436, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -5545,7 +6092,7 @@ __Pyx_GOTREF(__pyx_t_6); __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __pyx_t_1 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_6))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_6))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_6); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); @@ -5564,7 +6111,7 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __pyx_t_6 = NULL; __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -5574,20 +6121,40 @@ __pyx_t_10 = 1; } } - __pyx_t_1 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 437, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - if (__pyx_t_6) { - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_3, __pyx_n_u_surrogateescape}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[3] = {__pyx_t_6, __pyx_t_3, __pyx_n_u_surrogateescape}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-__pyx_t_10, 2+__pyx_t_10); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; + } else + #endif + { + __pyx_t_1 = PyTuple_New(2+__pyx_t_10); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + if (__pyx_t_6) { + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_6); __pyx_t_6 = NULL; + } + __Pyx_GIVEREF(__pyx_t_3); + PyTuple_SET_ITEM(__pyx_t_1, 0+__pyx_t_10, __pyx_t_3); + __Pyx_INCREF(__pyx_n_u_surrogateescape); + __Pyx_GIVEREF(__pyx_n_u_surrogateescape); + PyTuple_SET_ITEM(__pyx_t_1, 1+__pyx_t_10, __pyx_n_u_surrogateescape); + __pyx_t_3 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 437, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; } - __Pyx_GIVEREF(__pyx_t_3); - PyTuple_SET_ITEM(__pyx_t_1, 0+__pyx_t_10, __pyx_t_3); - __Pyx_INCREF(__pyx_n_u_surrogateescape); - __Pyx_GIVEREF(__pyx_n_u_surrogateescape); - PyTuple_SET_ITEM(__pyx_t_1, 1+__pyx_t_10, __pyx_n_u_surrogateescape); - __pyx_t_3 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 437, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __pyx_v_dbfile_b = __pyx_t_4; __pyx_t_4 = 0; @@ -5608,7 +6175,7 @@ * cm.callback(lambda: SQLITE_CHECK_RC(sqlite3_close(sqlite3_db), * SQLITE_OK, sqlite3_db)) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_open_v2(__pyx_t_12, (&__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OPEN_READWRITE, NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 438, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_open_v2(__pyx_t_12, (&__pyx_cur_scope->__pyx_v_sqlite3_db), SQLITE_OPEN_READWRITE, NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 438, __pyx_L8_error) /* "s3ql/deltadump.pyx":441 * SQLITE_OPEN_READWRITE, NULL), @@ -5622,7 +6189,7 @@ __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_lambda6, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 441, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -5636,15 +6203,35 @@ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 441, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 441, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -5656,7 +6243,7 @@ * SQLITE_OK, sqlite3_db) * */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_extended_result_codes(__pyx_cur_scope->__pyx_v_sqlite3_db, 1), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 443, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_extended_result_codes(__pyx_cur_scope->__pyx_v_sqlite3_db, 1), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 443, __pyx_L8_error) /* "s3ql/deltadump.pyx":447 * @@ -5665,13 +6252,13 @@ * val = db.get_val('PRAGMA %s' % pragma) * cmd = ('PRAGMA %s = %s' % (pragma, val)).encode('utf-8') */ - __pyx_t_4 = __pyx_tuple__8; __Pyx_INCREF(__pyx_t_4); __pyx_t_10 = 0; + __pyx_t_4 = __pyx_tuple__8; __Pyx_INCREF(__pyx_t_4); __pyx_t_13 = 0; for (;;) { - if (__pyx_t_10 >= 2) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_11 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_10); __Pyx_INCREF(__pyx_t_11); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 447, __pyx_L8_error) + if (__pyx_t_13 >= 2) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_11 = PyTuple_GET_ITEM(__pyx_t_4, __pyx_t_13); __Pyx_INCREF(__pyx_t_11); __pyx_t_13++; if (unlikely(0 < 0)) __PYX_ERR(0, 447, __pyx_L8_error) #else - __pyx_t_11 = PySequence_ITEM(__pyx_t_4, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 447, __pyx_L8_error) + __pyx_t_11 = PySequence_ITEM(__pyx_t_4, __pyx_t_13); __pyx_t_13++; if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 447, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_11); #endif __Pyx_XDECREF_SET(__pyx_v_pragma, ((PyObject*)__pyx_t_11)); @@ -5689,7 +6276,7 @@ __pyx_t_1 = PyUnicode_Format(__pyx_kp_u_PRAGMA_s, __pyx_v_pragma); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 448, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_3 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_6))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_6))) { __pyx_t_3 = PyMethod_GET_SELF(__pyx_t_6); if (likely(__pyx_t_3)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_6); @@ -5702,16 +6289,36 @@ __pyx_t_11 = __Pyx_PyObject_CallOneArg(__pyx_t_6, __pyx_t_1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_11); - } else { - __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 448, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_14); - __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_3); __pyx_t_3 = NULL; - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_14, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_11); - __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } else { + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_11 = __Pyx_PyFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_6)) { + PyObject *__pyx_temp[2] = {__pyx_t_3, __pyx_t_1}; + __pyx_t_11 = __Pyx_PyCFunction_FastCall(__pyx_t_6, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_3); __pyx_t_3 = 0; + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_3); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_3); __pyx_t_3 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_11 = __Pyx_PyObject_Call(__pyx_t_6, __pyx_t_14, NULL); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 448, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_11); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } } __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_XDECREF_SET(__pyx_v_val, __pyx_t_11); @@ -5757,7 +6364,7 @@ * try: * rc = sqlite3_step(stmt) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_15, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 450, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_prepare_v2(__pyx_cur_scope->__pyx_v_sqlite3_db, __pyx_t_15, -1, (&__pyx_cur_scope->__pyx_v_stmt), NULL), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 450, __pyx_L8_error) /* "s3ql/deltadump.pyx":452 * SQLITE_CHECK_RC(sqlite3_prepare_v2(sqlite3_db, cmd, -1, &stmt, NULL), @@ -5812,7 +6419,7 @@ * finally: * SQLITE_CHECK_RC(sqlite3_finalize(stmt), SQLITE_OK, sqlite3_db) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(__pyx_v_rc, SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 456, __pyx_L21_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(__pyx_v_rc, SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 456, __pyx_L21_error) } /* "s3ql/deltadump.pyx":458 @@ -5824,7 +6431,7 @@ */ /*finally:*/ { /*normal exit:*/{ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 458, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 458, __pyx_L8_error) goto __pyx_L22; } /*exception exit:*/{ @@ -5845,7 +6452,7 @@ __Pyx_XGOTREF(__pyx_t_21); __Pyx_XGOTREF(__pyx_t_22); __Pyx_XGOTREF(__pyx_t_23); - __pyx_t_13 = __pyx_lineno; __pyx_t_16 = __pyx_clineno; __pyx_t_17 = __pyx_filename; + __pyx_t_10 = __pyx_lineno; __pyx_t_16 = __pyx_clineno; __pyx_t_17 = __pyx_filename; { __pyx_t_24 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_finalize(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_24 == -1)) __PYX_ERR(0, 458, __pyx_L27_error) } @@ -5861,7 +6468,7 @@ __Pyx_XGIVEREF(__pyx_t_20); __Pyx_ErrRestore(__pyx_t_18, __pyx_t_19, __pyx_t_20); __pyx_t_18 = 0; __pyx_t_19 = 0; __pyx_t_20 = 0; __pyx_t_21 = 0; __pyx_t_22 = 0; __pyx_t_23 = 0; - __pyx_lineno = __pyx_t_13; __pyx_clineno = __pyx_t_16; __pyx_filename = __pyx_t_17; + __pyx_lineno = __pyx_t_10; __pyx_clineno = __pyx_t_16; __pyx_filename = __pyx_t_17; goto __pyx_L8_error; __pyx_L27_error:; __Pyx_PyThreadState_assign @@ -5912,7 +6519,7 @@ __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_1lambda7, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 462, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_14 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_14)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -5926,15 +6533,35 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 462, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_14); __pyx_t_14 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 462, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_14); __pyx_t_14 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 462, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -5961,7 +6588,7 @@ __pyx_t_1 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_2lambda8, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 466, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_1); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -5975,15 +6602,35 @@ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 466, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_14); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_1); - PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_1); - __pyx_t_1 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 466, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_1}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } else + #endif + { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_1); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_1); + __pyx_t_1 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 466, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -6000,7 +6647,7 @@ __pyx_t_14 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_3lambda9, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 467, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_14); __pyx_t_1 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_1 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_1)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -6014,15 +6661,35 @@ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_1); __pyx_t_1 = NULL; - __Pyx_GIVEREF(__pyx_t_14); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_14); - __pyx_t_14 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_t_14}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_1, __pyx_t_14}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_1); __pyx_t_1 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_1); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_1); __pyx_t_1 = NULL; + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_14); + __pyx_t_14 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 467, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -6034,8 +6701,8 @@ * cm.callback(lambda: free(int64_prev)) * */ - __pyx_t_10 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 470, __pyx_L8_error) - __pyx_t_26 = __pyx_f_4s3ql_9deltadump_calloc(((size_t)__pyx_t_10), (sizeof(int64_t))); if (unlikely(__pyx_t_26 == NULL)) __PYX_ERR(0, 470, __pyx_L8_error) + __pyx_t_13 = PyObject_Length(__pyx_v_columns); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 470, __pyx_L8_error) + __pyx_t_26 = __pyx_f_4s3ql_9deltadump_calloc(((size_t)__pyx_t_13), (sizeof(int64_t))); if (unlikely(__pyx_t_26 == NULL)) __PYX_ERR(0, 470, __pyx_L8_error) __pyx_cur_scope->__pyx_v_int64_prev = ((int64_t *)__pyx_t_26); /* "s3ql/deltadump.pyx":471 @@ -6050,7 +6717,7 @@ __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_4lambda10, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 471, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_14 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_14)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -6064,15 +6731,35 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_4); } else { - __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 471, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_14); __pyx_t_14 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_t_6}; + __pyx_t_4 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_1 = PyTuple_New(1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_t_14); __pyx_t_14 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_1, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_4 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_1, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 471, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; @@ -6089,29 +6776,29 @@ __pyx_t_4 = PyList_New(0); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 474, __pyx_L30_error) __Pyx_GOTREF(__pyx_t_4); if (likely(PyList_CheckExact(__pyx_v_columns)) || PyTuple_CheckExact(__pyx_v_columns)) { - __pyx_t_11 = __pyx_v_columns; __Pyx_INCREF(__pyx_t_11); __pyx_t_10 = 0; + __pyx_t_11 = __pyx_v_columns; __Pyx_INCREF(__pyx_t_11); __pyx_t_13 = 0; __pyx_t_27 = NULL; } else { - __pyx_t_10 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_columns); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 474, __pyx_L30_error) + __pyx_t_13 = -1; __pyx_t_11 = PyObject_GetIter(__pyx_v_columns); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 474, __pyx_L30_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_27 = Py_TYPE(__pyx_t_11)->tp_iternext; if (unlikely(!__pyx_t_27)) __PYX_ERR(0, 474, __pyx_L30_error) } for (;;) { if (likely(!__pyx_t_27)) { if (likely(PyList_CheckExact(__pyx_t_11))) { - if (__pyx_t_10 >= PyList_GET_SIZE(__pyx_t_11)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_1 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_1); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 474, __pyx_L30_error) + if (__pyx_t_13 >= PyList_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_1 = PyList_GET_ITEM(__pyx_t_11, __pyx_t_13); __Pyx_INCREF(__pyx_t_1); __pyx_t_13++; if (unlikely(0 < 0)) __PYX_ERR(0, 474, __pyx_L30_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_11, __pyx_t_13); __pyx_t_13++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) __Pyx_GOTREF(__pyx_t_1); #endif } else { - if (__pyx_t_10 >= PyTuple_GET_SIZE(__pyx_t_11)) break; - #if CYTHON_COMPILING_IN_CPYTHON - __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_10); __Pyx_INCREF(__pyx_t_1); __pyx_t_10++; if (unlikely(0 < 0)) __PYX_ERR(0, 474, __pyx_L30_error) + if (__pyx_t_13 >= PyTuple_GET_SIZE(__pyx_t_11)) break; + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS + __pyx_t_1 = PyTuple_GET_ITEM(__pyx_t_11, __pyx_t_13); __Pyx_INCREF(__pyx_t_1); __pyx_t_13++; if (unlikely(0 < 0)) __PYX_ERR(0, 474, __pyx_L30_error) #else - __pyx_t_1 = PySequence_ITEM(__pyx_t_11, __pyx_t_10); __pyx_t_10++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) + __pyx_t_1 = PySequence_ITEM(__pyx_t_11, __pyx_t_13); __pyx_t_13++; if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 474, __pyx_L30_error) __Pyx_GOTREF(__pyx_t_1); #endif } @@ -6236,7 +6923,7 @@ __pyx_t_4 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_5lambda11, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 480, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -6250,15 +6937,35 @@ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else { - __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 480, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_14); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 480, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 480, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6303,7 +7010,7 @@ __pyx_t_14 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_6lambda12, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 487, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_14); __pyx_t_4 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_4)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -6317,15 +7024,35 @@ __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_GOTREF(__pyx_t_1); } else { - __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 487, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; - __Pyx_GIVEREF(__pyx_t_14); - PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_14); - __pyx_t_14 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_14}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_14}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(1+1); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_14); + PyTuple_SET_ITEM(__pyx_t_6, 0+1, __pyx_t_14); + __pyx_t_14 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 487, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6370,7 +7097,7 @@ __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_7lambda13, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 494, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_14 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_14 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_14)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -6384,15 +7111,35 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 494, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_14); __pyx_t_14 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_14, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_14); __pyx_t_14 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_14); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_14); __pyx_t_14 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 494, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6423,7 +7170,7 @@ __pyx_t_4 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_8lambda14, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 498, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_4); __pyx_t_6 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_11))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_11))) { __pyx_t_6 = PyMethod_GET_SELF(__pyx_t_11); if (likely(__pyx_t_6)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_11); @@ -6437,15 +7184,35 @@ __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; __Pyx_GOTREF(__pyx_t_1); } else { - __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 498, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_14); - __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; - __Pyx_GIVEREF(__pyx_t_4); - PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_4); - __pyx_t_4 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_11)) { + PyObject *__pyx_temp[2] = {__pyx_t_6, __pyx_t_4}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_11, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_6); __pyx_t_6 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } else + #endif + { + __pyx_t_14 = PyTuple_New(1+1); if (unlikely(!__pyx_t_14)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_14); + __Pyx_GIVEREF(__pyx_t_6); PyTuple_SET_ITEM(__pyx_t_14, 0, __pyx_t_6); __pyx_t_6 = NULL; + __Pyx_GIVEREF(__pyx_t_4); + PyTuple_SET_ITEM(__pyx_t_14, 0+1, __pyx_t_4); + __pyx_t_4 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_11, __pyx_t_14, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 498, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; + } } __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6474,34 +7241,54 @@ __pyx_t_11 = __Pyx_PyInt_From_int64_t(__pyx_v_row_count); if (unlikely(!__pyx_t_11)) __PYX_ERR(0, 500, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_11); __pyx_t_4 = NULL; - __pyx_t_10 = 0; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_14))) { + __pyx_t_16 = 0; + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_14))) { __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_14); if (likely(__pyx_t_4)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_14); __Pyx_INCREF(__pyx_t_4); __Pyx_INCREF(function); __Pyx_DECREF_SET(__pyx_t_14, function); - __pyx_t_10 = 1; + __pyx_t_16 = 1; } } - __pyx_t_6 = PyTuple_New(3+__pyx_t_10); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 500, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_6); - if (__pyx_t_4) { - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_14)) { + PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_kp_u_load_table_s_reading_d_rows, __pyx_v_table, __pyx_t_11}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_14, __pyx_temp+1-__pyx_t_16, 3+__pyx_t_16); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_14)) { + PyObject *__pyx_temp[4] = {__pyx_t_4, __pyx_kp_u_load_table_s_reading_d_rows, __pyx_v_table, __pyx_t_11}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_14, __pyx_temp+1-__pyx_t_16, 3+__pyx_t_16); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_11); __pyx_t_11 = 0; + } else + #endif + { + __pyx_t_6 = PyTuple_New(3+__pyx_t_16); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_6); + if (__pyx_t_4) { + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_t_4); __pyx_t_4 = NULL; + } + __Pyx_INCREF(__pyx_kp_u_load_table_s_reading_d_rows); + __Pyx_GIVEREF(__pyx_kp_u_load_table_s_reading_d_rows); + PyTuple_SET_ITEM(__pyx_t_6, 0+__pyx_t_16, __pyx_kp_u_load_table_s_reading_d_rows); + __Pyx_INCREF(__pyx_v_table); + __Pyx_GIVEREF(__pyx_v_table); + PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_16, __pyx_v_table); + __Pyx_GIVEREF(__pyx_t_11); + PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_16, __pyx_t_11); + __pyx_t_11 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_14, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 500, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; } - __Pyx_INCREF(__pyx_kp_u_load_table_s_reading_d_rows); - __Pyx_GIVEREF(__pyx_kp_u_load_table_s_reading_d_rows); - PyTuple_SET_ITEM(__pyx_t_6, 0+__pyx_t_10, __pyx_kp_u_load_table_s_reading_d_rows); - __Pyx_INCREF(__pyx_v_table); - __Pyx_GIVEREF(__pyx_v_table); - PyTuple_SET_ITEM(__pyx_t_6, 1+__pyx_t_10, __pyx_v_table); - __Pyx_GIVEREF(__pyx_t_11); - PyTuple_SET_ITEM(__pyx_t_6, 2+__pyx_t_10, __pyx_t_11); - __pyx_t_11 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_14, __pyx_t_6, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 500, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6526,7 +7313,7 @@ __pyx_t_6 = __Pyx_CyFunction_NewEx(&__pyx_mdef_4s3ql_9deltadump_10load_table_9lambda15, 0, __pyx_n_s_load_table_locals_lambda, ((PyObject*)__pyx_cur_scope), __pyx_n_s_s3ql_deltadump, __pyx_d, NULL); if (unlikely(!__pyx_t_6)) __PYX_ERR(0, 504, __pyx_L8_error) __Pyx_GOTREF(__pyx_t_6); __pyx_t_11 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && likely(PyMethod_Check(__pyx_t_14))) { + if (CYTHON_UNPACK_METHODS && likely(PyMethod_Check(__pyx_t_14))) { __pyx_t_11 = PyMethod_GET_SELF(__pyx_t_14); if (likely(__pyx_t_11)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_14); @@ -6540,15 +7327,35 @@ __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; __Pyx_GOTREF(__pyx_t_1); } else { - __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 504, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_4); - __Pyx_GIVEREF(__pyx_t_11); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_11); __pyx_t_11 = NULL; - __Pyx_GIVEREF(__pyx_t_6); - PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_t_6); - __pyx_t_6 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_14, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 504, __pyx_L8_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_14)) { + PyObject *__pyx_temp[2] = {__pyx_t_11, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_14, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_14)) { + PyObject *__pyx_temp[2] = {__pyx_t_11, __pyx_t_6}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_14, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_XDECREF(__pyx_t_11); __pyx_t_11 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0; + } else + #endif + { + __pyx_t_4 = PyTuple_New(1+1); if (unlikely(!__pyx_t_4)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_4); + __Pyx_GIVEREF(__pyx_t_11); PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_t_11); __pyx_t_11 = NULL; + __Pyx_GIVEREF(__pyx_t_6); + PyTuple_SET_ITEM(__pyx_t_4, 0+1, __pyx_t_6); + __pyx_t_6 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_14, __pyx_t_4, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 504, __pyx_L8_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0; + } } __Pyx_DECREF(__pyx_t_14); __pyx_t_14 = 0; __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0; @@ -6580,8 +7387,8 @@ * if col_types[j] == _INTEGER: * read_integer(&int64, fp) */ - __pyx_t_13 = __pyx_v_col_count; - for (__pyx_t_24 = 0; __pyx_t_24 < __pyx_t_13; __pyx_t_24+=1) { + __pyx_t_10 = __pyx_v_col_count; + for (__pyx_t_24 = 0; __pyx_t_24 < __pyx_t_10; __pyx_t_24+=1) { __pyx_v_j = __pyx_t_24; /* "s3ql/deltadump.pyx":511 @@ -6936,7 +7743,7 @@ * SQLITE_CHECK_RC(sqlite3_reset(stmt), SQLITE_OK, sqlite3_db) * */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 547, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 547, __pyx_L8_error) /* "s3ql/deltadump.pyx":548 * @@ -6945,7 +7752,7 @@ * * # Commit every once in a while */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 548, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 548, __pyx_L8_error) /* "s3ql/deltadump.pyx":551 * @@ -6973,7 +7780,7 @@ * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) * SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 555, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 555, __pyx_L8_error) /* "s3ql/deltadump.pyx":556 * # without an active transaction. @@ -6982,7 +7789,7 @@ * SQLITE_CHECK_RC(sqlite3_reset(commit_stmt), SQLITE_OK, sqlite3_db) * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 556, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_step(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_DONE, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 556, __pyx_L8_error) /* "s3ql/deltadump.pyx":557 * SQLITE_CHECK_RC(sqlite3_step(commit_stmt), SQLITE_DONE, sqlite3_db) @@ -6991,7 +7798,7 @@ * SQLITE_CHECK_RC(sqlite3_reset(begin_stmt), SQLITE_OK, sqlite3_db) * */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 557, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_commit_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 557, __pyx_L8_error) /* "s3ql/deltadump.pyx":558 * SQLITE_CHECK_RC(sqlite3_step(begin_stmt), SQLITE_DONE, sqlite3_db) @@ -7000,7 +7807,7 @@ * * cdef inline int write_integer(int64_t int64, FILE * fp) except -1: */ - __pyx_t_13 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_13 == -1)) __PYX_ERR(0, 558, __pyx_L8_error) + __pyx_t_10 = __pyx_f_4s3ql_9deltadump_SQLITE_CHECK_RC(sqlite3_reset(__pyx_cur_scope->__pyx_v_begin_stmt), SQLITE_OK, __pyx_cur_scope->__pyx_v_sqlite3_db); if (unlikely(__pyx_t_10 == -1)) __PYX_ERR(0, 558, __pyx_L8_error) /* "s3ql/deltadump.pyx":551 * @@ -8300,6 +9107,7 @@ if (__Pyx_check_binary_version() < 0) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_empty_tuple = PyTuple_New(0); if (unlikely(!__pyx_empty_tuple)) __PYX_ERR(0, 1, __pyx_L1_error) __pyx_empty_bytes = PyBytes_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_bytes)) __PYX_ERR(0, 1, __pyx_L1_error) + __pyx_empty_unicode = PyUnicode_FromStringAndSize("", 0); if (unlikely(!__pyx_empty_unicode)) __PYX_ERR(0, 1, __pyx_L1_error) #ifdef __Pyx_CyFunction_USED if (__pyx_CyFunction_init() < 0) __PYX_ERR(0, 1, __pyx_L1_error) #endif @@ -8486,7 +9294,7 @@ __pyx_t_2 = __Pyx_GetModuleGlobalName(__pyx_n_s_name); if (unlikely(!__pyx_t_2)) __PYX_ERR(0, 86, __pyx_L1_error) __Pyx_GOTREF(__pyx_t_2); __pyx_t_4 = NULL; - if (CYTHON_COMPILING_IN_CPYTHON && unlikely(PyMethod_Check(__pyx_t_3))) { + if (CYTHON_UNPACK_METHODS && unlikely(PyMethod_Check(__pyx_t_3))) { __pyx_t_4 = PyMethod_GET_SELF(__pyx_t_3); if (likely(__pyx_t_4)) { PyObject* function = PyMethod_GET_FUNCTION(__pyx_t_3); @@ -8500,15 +9308,35 @@ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; __Pyx_GOTREF(__pyx_t_1); } else { - __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 86, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_5); - __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __pyx_t_4 = NULL; - __Pyx_GIVEREF(__pyx_t_2); - PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_2); - __pyx_t_2 = 0; - __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) - __Pyx_GOTREF(__pyx_t_1); - __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + #if CYTHON_FAST_PYCALL + if (PyFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_2}; + __pyx_t_1 = __Pyx_PyFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else + #endif + #if CYTHON_FAST_PYCCALL + if (__Pyx_PyFastCFunction_Check(__pyx_t_3)) { + PyObject *__pyx_temp[2] = {__pyx_t_4, __pyx_t_2}; + __pyx_t_1 = __Pyx_PyCFunction_FastCall(__pyx_t_3, __pyx_temp+1-1, 1+1); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_XDECREF(__pyx_t_4); __pyx_t_4 = 0; + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0; + } else + #endif + { + __pyx_t_5 = PyTuple_New(1+1); if (unlikely(!__pyx_t_5)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_5); + __Pyx_GIVEREF(__pyx_t_4); PyTuple_SET_ITEM(__pyx_t_5, 0, __pyx_t_4); __pyx_t_4 = NULL; + __Pyx_GIVEREF(__pyx_t_2); + PyTuple_SET_ITEM(__pyx_t_5, 0+1, __pyx_t_2); + __pyx_t_2 = 0; + __pyx_t_1 = __Pyx_PyObject_Call(__pyx_t_3, __pyx_t_5, NULL); if (unlikely(!__pyx_t_1)) __PYX_ERR(0, 86, __pyx_L1_error) + __Pyx_GOTREF(__pyx_t_1); + __Pyx_DECREF(__pyx_t_5); __pyx_t_5 = 0; + } } __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0; if (PyDict_SetItem(__pyx_d, __pyx_n_s_log, __pyx_t_1) < 0) __PYX_ERR(0, 86, __pyx_L1_error) @@ -8715,6 +9543,7 @@ } /* --- Runtime support code --- */ +/* Refnanny */ #if CYTHON_REFNANNY static __Pyx_RefNannyAPIStruct *__Pyx_RefNannyImportAPI(const char *modname) { PyObject *m = NULL, *p = NULL; @@ -8731,6 +9560,7 @@ } #endif +/* GetBuiltinName */ static PyObject *__Pyx_GetBuiltinName(PyObject *name) { PyObject* result = __Pyx_PyObject_GetAttrStr(__pyx_b, name); if (unlikely(!result)) { @@ -8744,6 +9574,145 @@ return result; } +/* PyFunctionFastCall */ +#if CYTHON_FAST_PYCALL +#include "frameobject.h" +static PyObject* __Pyx_PyFunction_FastCallNoKw(PyCodeObject *co, PyObject **args, Py_ssize_t na, + PyObject *globals) { + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + return result; +} +#if 1 || PY_VERSION_HEX < 0x030600B1 +static PyObject *__Pyx_PyFunction_FastCallDict(PyObject *func, PyObject **args, int nargs, PyObject *kwargs) { + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *closure; +#if PY_MAJOR_VERSION >= 3 + PyObject *kwdefs; +#endif + PyObject *kwtuple, **k; + PyObject **d; + Py_ssize_t nd; + Py_ssize_t nk; + PyObject *result; + assert(kwargs == NULL || PyDict_Check(kwargs)); + nk = kwargs ? PyDict_Size(kwargs) : 0; + if (Py_EnterRecursiveCall((char*)" while calling a Python object")) { + return NULL; + } + if ( +#if PY_MAJOR_VERSION >= 3 + co->co_kwonlyargcount == 0 && +#endif + likely(kwargs == NULL || nk == 0) && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { + if (argdefs == NULL && co->co_argcount == nargs) { + result = __Pyx_PyFunction_FastCallNoKw(co, args, nargs, globals); + goto done; + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + result =__Pyx_PyFunction_FastCallNoKw(co, args, Py_SIZE(argdefs), globals); + goto done; + } + } + if (kwargs != NULL) { + Py_ssize_t pos, i; + kwtuple = PyTuple_New(2 * nk); + if (kwtuple == NULL) { + result = NULL; + goto done; + } + k = &PyTuple_GET_ITEM(kwtuple, 0); + pos = i = 0; + while (PyDict_Next(kwargs, &pos, &k[i], &k[i+1])) { + Py_INCREF(k[i]); + Py_INCREF(k[i+1]); + i += 2; + } + nk = i / 2; + } + else { + kwtuple = NULL; + k = NULL; + } + closure = PyFunction_GET_CLOSURE(func); +#if PY_MAJOR_VERSION >= 3 + kwdefs = PyFunction_GET_KW_DEFAULTS(func); +#endif + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } +#if PY_MAJOR_VERSION >= 3 + result = PyEval_EvalCodeEx((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, kwdefs, closure); +#else + result = PyEval_EvalCodeEx(co, globals, (PyObject *)NULL, + args, nargs, + k, (int)nk, + d, (int)nd, closure); +#endif + Py_XDECREF(kwtuple); +done: + Py_LeaveRecursiveCall(); + return result; +} +#endif // CPython < 3.6 +#endif // CYTHON_FAST_PYCALL + +/* PyCFunctionFastCall */ +#if CYTHON_FAST_PYCCALL +static CYTHON_INLINE PyObject * __Pyx_PyCFunction_FastCall(PyObject *func_obj, PyObject **args, Py_ssize_t nargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + assert(PyCFunction_Check(func)); + assert(METH_FASTCALL == (PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST))); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + return (*((__Pyx_PyCFunctionFast)meth)) (self, args, nargs, NULL); +} +#endif // CYTHON_FAST_PYCCALL + +/* PyObjectCall */ #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_Call(PyObject *func, PyObject *arg, PyObject *kw) { PyObject *result; @@ -8763,7 +9732,8 @@ } #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* PyErrFetchRestore */ +#if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx_ErrRestoreInState(PyThreadState *tstate, PyObject *type, PyObject *value, PyObject *tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; tmp_type = tstate->curexc_type; @@ -8786,6 +9756,7 @@ } #endif +/* RaiseException */ #if PY_MAJOR_VERSION < 3 static void __Pyx_Raise(PyObject *type, PyObject *value, PyObject *tb, CYTHON_UNUSED PyObject *cause) { @@ -8948,9 +9919,10 @@ } #endif -static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { +/* GetModuleGlobalName */ + static CYTHON_INLINE PyObject *__Pyx_GetModuleGlobalName(PyObject *name) { PyObject *result; -#if CYTHON_COMPILING_IN_CPYTHON +#if !CYTHON_AVOID_BORROWED_REFS result = PyDict_GetItem(__pyx_d, name); if (likely(result)) { Py_INCREF(result); @@ -8965,7 +9937,8 @@ return result; } -#if CYTHON_COMPILING_IN_CPYTHON +/* PyObjectCallMethO */ + #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallMethO(PyObject *func, PyObject *arg) { PyObject *self, *result; PyCFunction cfunc; @@ -8984,7 +9957,8 @@ } #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* PyObjectCallOneArg */ + #if CYTHON_COMPILING_IN_CPYTHON static PyObject* __Pyx__PyObject_CallOneArg(PyObject *func, PyObject *arg) { PyObject *result; PyObject *args = PyTuple_New(1); @@ -8996,6 +9970,11 @@ return result; } static CYTHON_INLINE PyObject* __Pyx_PyObject_CallOneArg(PyObject *func, PyObject *arg) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, &arg, 1); + } +#endif #ifdef __Pyx_CyFunction_USED if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { #else @@ -9003,6 +9982,10 @@ #endif if (likely(PyCFunction_GET_FLAGS(func) & METH_O)) { return __Pyx_PyObject_CallMethO(func, arg); +#if CYTHON_FAST_PYCCALL + } else if (PyCFunction_GET_FLAGS(func) & METH_FASTCALL) { + return __Pyx_PyCFunction_FastCall(func, &arg, 1); +#endif } } return __Pyx__PyObject_CallOneArg(func, arg); @@ -9018,7 +10001,8 @@ } #endif -static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { +/* GetItemInt */ + static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Generic(PyObject *o, PyObject* j) { PyObject *r; if (!j) return NULL; r = PyObject_GetItem(o, j); @@ -9028,7 +10012,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_List_Fast(PyObject *o, Py_ssize_t i, CYTHON_NCP_UNUSED int wraparound, CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (wraparound & unlikely(i < 0)) i += PyList_GET_SIZE(o); if ((!boundscheck) || likely((0 <= i) & (i < PyList_GET_SIZE(o)))) { PyObject *r = PyList_GET_ITEM(o, i); @@ -9043,7 +10027,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Tuple_Fast(PyObject *o, Py_ssize_t i, CYTHON_NCP_UNUSED int wraparound, CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS if (wraparound & unlikely(i < 0)) i += PyTuple_GET_SIZE(o); if ((!boundscheck) || likely((0 <= i) & (i < PyTuple_GET_SIZE(o)))) { PyObject *r = PyTuple_GET_ITEM(o, i); @@ -9058,7 +10042,7 @@ static CYTHON_INLINE PyObject *__Pyx_GetItemInt_Fast(PyObject *o, Py_ssize_t i, int is_list, CYTHON_NCP_UNUSED int wraparound, CYTHON_NCP_UNUSED int boundscheck) { -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS && CYTHON_USE_TYPE_SLOTS if (is_list || PyList_CheckExact(o)) { Py_ssize_t n = ((!wraparound) | likely(i >= 0)) ? i : i + PyList_GET_SIZE(o); if ((!boundscheck) || (likely((n >= 0) & (n < PyList_GET_SIZE(o))))) { @@ -9098,8 +10082,14 @@ return __Pyx_GetItemInt_Generic(o, PyInt_FromSsize_t(i)); } -#if CYTHON_COMPILING_IN_CPYTHON +/* PyObjectCallNoArg */ + #if CYTHON_COMPILING_IN_CPYTHON static CYTHON_INLINE PyObject* __Pyx_PyObject_CallNoArg(PyObject *func) { +#if CYTHON_FAST_PYCALL + if (PyFunction_Check(func)) { + return __Pyx_PyFunction_FastCall(func, NULL, 0); + } +#endif #ifdef __Pyx_CyFunction_USED if (likely(PyCFunction_Check(func) || PyObject_TypeCheck(func, __pyx_CyFunctionType))) { #else @@ -9113,7 +10103,8 @@ } #endif -static void __Pyx_RaiseArgtupleInvalid( +/* RaiseArgTupleInvalid */ + static void __Pyx_RaiseArgtupleInvalid( const char* func_name, int exact, Py_ssize_t num_min, @@ -9138,7 +10129,8 @@ (num_expected == 1) ? "" : "s", num_found); } -static void __Pyx_RaiseDoubleKeywordsError( +/* RaiseDoubleKeywords */ + static void __Pyx_RaiseDoubleKeywordsError( const char* func_name, PyObject* kw_name) { @@ -9151,7 +10143,8 @@ #endif } -static int __Pyx_ParseOptionalKeywords( +/* ParseKeywords */ + static int __Pyx_ParseOptionalKeywords( PyObject *kwds, PyObject **argnames[], PyObject *kwds2, @@ -9252,7 +10245,8 @@ return -1; } -static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { +/* BytesEquals */ + static CYTHON_INLINE int __Pyx_PyBytes_Equals(PyObject* s1, PyObject* s2, int equals) { #if CYTHON_COMPILING_IN_PYPY return PyObject_RichCompareBool(s1, s2, equals); #else @@ -9289,7 +10283,8 @@ #endif } -static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { +/* UnicodeEquals */ + static CYTHON_INLINE int __Pyx_PyUnicode_Equals(PyObject* s1, PyObject* s2, int equals) { #if CYTHON_COMPILING_IN_PYPY return PyObject_RichCompareBool(s1, s2, equals); #else @@ -9372,7 +10367,8 @@ #endif } -static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { +/* FetchCommonType */ + static PyTypeObject* __Pyx_FetchCommonType(PyTypeObject* type) { PyObject* fake_module; PyTypeObject* cached_type = NULL; fake_module = PyImport_AddModule((char*) "_cython_" CYTHON_ABI); @@ -9410,7 +10406,8 @@ goto done; } -static PyObject * +/* CythonFunction */ + static PyObject * __Pyx_CyFunction_get_doc(__pyx_CyFunctionObject *op, CYTHON_UNUSED void *closure) { if (unlikely(op->func_doc == NULL)) { @@ -9567,7 +10564,7 @@ PyObject *res = op->defaults_getter((PyObject *) op); if (unlikely(!res)) return -1; - #if CYTHON_COMPILING_IN_CPYTHON + #if CYTHON_ASSUME_SAFE_MACROS && !CYTHON_AVOID_BORROWED_REFS op->defaults_tuple = PyTuple_GET_ITEM(res, 0); Py_INCREF(op->defaults_tuple); op->defaults_kwdict = PyTuple_GET_ITEM(res, 1); @@ -9826,11 +10823,9 @@ PyString_AsString(op->func_qualname), (void *)op); #endif } -#if CYTHON_COMPILING_IN_PYPY -static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { +static PyObject * __Pyx_CyFunction_CallMethod(PyObject *func, PyObject *self, PyObject *arg, PyObject *kw) { PyCFunctionObject* f = (PyCFunctionObject*)func; PyCFunction meth = f->m_ml->ml_meth; - PyObject *self = f->m_self; Py_ssize_t size; switch (f->m_ml->ml_flags & (METH_VARARGS | METH_KEYWORDS | METH_NOARGS | METH_O)) { case METH_VARARGS: @@ -9876,11 +10871,32 @@ f->m_ml->ml_name); return NULL; } -#else -static PyObject * __Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { - return PyCFunction_Call(func, arg, kw); +static CYTHON_INLINE PyObject *__Pyx_CyFunction_Call(PyObject *func, PyObject *arg, PyObject *kw) { + return __Pyx_CyFunction_CallMethod(func, ((PyCFunctionObject*)func)->m_self, arg, kw); +} +static PyObject *__Pyx_CyFunction_CallAsMethod(PyObject *func, PyObject *args, PyObject *kw) { + PyObject *result; + __pyx_CyFunctionObject *cyfunc = (__pyx_CyFunctionObject *) func; + if ((cyfunc->flags & __Pyx_CYFUNCTION_CCLASS) && !(cyfunc->flags & __Pyx_CYFUNCTION_STATICMETHOD)) { + Py_ssize_t argc; + PyObject *new_args; + PyObject *self; + argc = PyTuple_GET_SIZE(args); + new_args = PyTuple_GetSlice(args, 1, argc); + if (unlikely(!new_args)) + return NULL; + self = PyTuple_GetItem(args, 0); + if (unlikely(!self)) { + Py_DECREF(new_args); + return NULL; + } + result = __Pyx_CyFunction_CallMethod(func, self, new_args, kw); + Py_DECREF(new_args); + } else { + result = __Pyx_CyFunction_Call(func, args, kw); + } + return result; } -#endif static PyTypeObject __pyx_CyFunctionType_type = { PyVarObject_HEAD_INIT(0, 0) "cython_function_or_method", @@ -9900,7 +10916,7 @@ 0, 0, 0, - __Pyx_CyFunction_Call, + __Pyx_CyFunction_CallAsMethod, 0, 0, 0, @@ -9942,9 +10958,6 @@ #endif }; static int __pyx_CyFunction_init(void) { -#if !CYTHON_COMPILING_IN_PYPY - __pyx_CyFunctionType_type.tp_call = PyCFunction_Call; -#endif __pyx_CyFunctionType = __Pyx_FetchCommonType(&__pyx_CyFunctionType_type); if (__pyx_CyFunctionType == NULL) { return -1; @@ -9976,7 +10989,8 @@ Py_INCREF(dict); } -#if CYTHON_COMPILING_IN_CPYTHON +/* SaveResetException */ + #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSave(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { *type = tstate->exc_type; *value = tstate->exc_value; @@ -9999,13 +11013,14 @@ } #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* GetException */ + #if CYTHON_FAST_THREAD_STATE static int __Pyx__GetException(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { #else static int __Pyx_GetException(PyObject **type, PyObject **value, PyObject **tb) { #endif PyObject *local_type, *local_value, *local_tb; -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_FAST_THREAD_STATE PyObject *tmp_type, *tmp_value, *tmp_tb; local_type = tstate->curexc_type; local_value = tstate->curexc_value; @@ -10017,7 +11032,7 @@ PyErr_Fetch(&local_type, &local_value, &local_tb); #endif PyErr_NormalizeException(&local_type, &local_value, &local_tb); -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_FAST_THREAD_STATE if (unlikely(tstate->curexc_type)) #else if (unlikely(PyErr_Occurred())) @@ -10035,7 +11050,7 @@ *type = local_type; *value = local_value; *tb = local_tb; -#if CYTHON_COMPILING_IN_CPYTHON +#if CYTHON_FAST_THREAD_STATE tmp_type = tstate->exc_type; tmp_value = tstate->exc_value; tmp_tb = tstate->exc_traceback; @@ -10059,7 +11074,8 @@ return -1; } -#if CYTHON_COMPILING_IN_CPYTHON +/* SwapException */ + #if CYTHON_FAST_THREAD_STATE static CYTHON_INLINE void __Pyx__ExceptionSwap(PyThreadState *tstate, PyObject **type, PyObject **value, PyObject **tb) { PyObject *tmp_type, *tmp_value, *tmp_tb; tmp_type = tstate->exc_type; @@ -10083,7 +11099,8 @@ } #endif -#if CYTHON_COMPILING_IN_CPYTHON +/* PyIntBinop */ + #if !CYTHON_COMPILING_IN_PYPY static PyObject* __Pyx_PyInt_EqObjC(PyObject *op1, PyObject *op2, CYTHON_UNUSED long intval, CYTHON_UNUSED int inplace) { if (op1 == op2) { Py_RETURN_TRUE; @@ -10099,7 +11116,7 @@ } } #endif - #if CYTHON_USE_PYLONG_INTERNALS && PY_MAJOR_VERSION >= 3 + #if CYTHON_USE_PYLONG_INTERNALS if (likely(PyLong_CheckExact(op1))) { const long b = intval; long a; @@ -10167,7 +11184,8 @@ } #endif -static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { +/* Import */ + static PyObject *__Pyx_Import(PyObject *name, PyObject *from_list, int level) { PyObject *empty_list = 0; PyObject *module = 0; PyObject *global_dict = 0; @@ -10240,7 +11258,8 @@ return module; } -static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { +/* ImportFrom */ + static PyObject* __Pyx_ImportFrom(PyObject* module, PyObject* name) { PyObject* value = __Pyx_PyObject_GetAttrStr(module, name); if (unlikely(!value) && PyErr_ExceptionMatches(PyExc_AttributeError)) { PyErr_Format(PyExc_ImportError, @@ -10253,7 +11272,8 @@ return value; } -static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { +/* CodeObjectCache */ + static int __pyx_bisect_code_objects(__Pyx_CodeObjectCacheEntry* entries, int count, int code_line) { int start = 0, mid = 0, end = count - 1; if (end >= 0 && code_line > entries[end].code_line) { return count; @@ -10332,7 +11352,8 @@ Py_INCREF(code_object); } -#include "compile.h" +/* AddTraceback */ + #include "compile.h" #include "frameobject.h" #include "traceback.h" static PyCodeObject* __Pyx_CreateCodeObjectForTraceback( @@ -10405,14 +11426,15 @@ 0 /*PyObject *locals*/ ); if (!py_frame) goto bad; - py_frame->f_lineno = py_line; + __Pyx_PyFrame_SetLineNumber(py_frame, py_line); PyTraceBack_Here(py_frame); bad: Py_XDECREF(py_code); Py_XDECREF(py_frame); } -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int(int value) { const int neg_one = (int) -1, const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { @@ -10420,14 +11442,18 @@ return PyInt_FromLong((long) value); } else if (sizeof(int) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif } } else { if (sizeof(int) <= sizeof(long)) { return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif } } { @@ -10438,7 +11464,8 @@ } } -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_long(long value) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { @@ -10446,14 +11473,18 @@ return PyInt_FromLong((long) value); } else if (sizeof(long) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif } } else { if (sizeof(long) <= sizeof(long)) { return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif } } { @@ -10464,7 +11495,8 @@ } } -#define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ +/* CIntFromPyVerify */ + #define __PYX_VERIFY_RETURN_INT(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 0) #define __PYX_VERIFY_RETURN_INT_EXC(target_type, func_type, func_value)\ __PYX__VERIFY_RETURN_INT(target_type, func_type, func_value, 1) @@ -10485,7 +11517,8 @@ return (target_type) value;\ } -static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int64_t(int64_t value) { +/* CIntToPy */ + static CYTHON_INLINE PyObject* __Pyx_PyInt_From_int64_t(int64_t value) { const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; const int is_unsigned = neg_one > const_zero; if (is_unsigned) { @@ -10493,14 +11526,18 @@ return PyInt_FromLong((long) value); } else if (sizeof(int64_t) <= sizeof(unsigned long)) { return PyLong_FromUnsignedLong((unsigned long) value); +#ifdef HAVE_LONG_LONG } else if (sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG)) { return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG) value); +#endif } } else { if (sizeof(int64_t) <= sizeof(long)) { return PyInt_FromLong((long) value); +#ifdef HAVE_LONG_LONG } else if (sizeof(int64_t) <= sizeof(PY_LONG_LONG)) { return PyLong_FromLongLong((PY_LONG_LONG) value); +#endif } } { @@ -10511,7 +11548,8 @@ } } -static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { +/* CIntFromPy */ + static CYTHON_INLINE int __Pyx_PyInt_As_int(PyObject *x) { const int neg_one = (int) -1, const_zero = (int) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -10578,15 +11616,17 @@ #endif if (sizeof(int) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(int, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(int) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(int, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (int) 0; - case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) -digits[0]) + case -1: __PYX_VERIFY_RETURN_INT(int, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(int, digit, +digits[0]) case -2: if (8 * sizeof(int) - 1 > 1 * PyLong_SHIFT) { @@ -10646,8 +11686,10 @@ #endif if (sizeof(int) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(int, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(int) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(int, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif } } { @@ -10695,7 +11737,8 @@ return (int) -1; } -static CYTHON_INLINE int64_t __Pyx_PyInt_As_int64_t(PyObject *x) { +/* CIntFromPy */ + static CYTHON_INLINE int64_t __Pyx_PyInt_As_int64_t(PyObject *x) { const int64_t neg_one = (int64_t) -1, const_zero = (int64_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -10762,15 +11805,17 @@ #endif if (sizeof(int64_t) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(int64_t) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(int64_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (int64_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(int64_t, sdigit, (sdigit) -digits[0]) + case -1: __PYX_VERIFY_RETURN_INT(int64_t, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(int64_t, digit, +digits[0]) case -2: if (8 * sizeof(int64_t) - 1 > 1 * PyLong_SHIFT) { @@ -10830,8 +11875,10 @@ #endif if (sizeof(int64_t) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(int64_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(int64_t) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(int64_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif } } { @@ -10879,7 +11926,8 @@ return (int64_t) -1; } -static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { +/* CIntFromPy */ + static CYTHON_INLINE size_t __Pyx_PyInt_As_size_t(PyObject *x) { const size_t neg_one = (size_t) -1, const_zero = (size_t) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -10946,15 +11994,17 @@ #endif if (sizeof(size_t) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(size_t) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(size_t, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (size_t) 0; - case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) -digits[0]) + case -1: __PYX_VERIFY_RETURN_INT(size_t, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(size_t, digit, +digits[0]) case -2: if (8 * sizeof(size_t) - 1 > 1 * PyLong_SHIFT) { @@ -11014,8 +12064,10 @@ #endif if (sizeof(size_t) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(size_t, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(size_t) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(size_t, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif } } { @@ -11063,7 +12115,8 @@ return (size_t) -1; } -static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { +/* CIntFromPy */ + static CYTHON_INLINE long __Pyx_PyInt_As_long(PyObject *x) { const long neg_one = (long) -1, const_zero = (long) 0; const int is_unsigned = neg_one > const_zero; #if PY_MAJOR_VERSION < 3 @@ -11130,15 +12183,17 @@ #endif if (sizeof(long) <= sizeof(unsigned long)) { __PYX_VERIFY_RETURN_INT_EXC(long, unsigned long, PyLong_AsUnsignedLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(long) <= sizeof(unsigned PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(long, unsigned PY_LONG_LONG, PyLong_AsUnsignedLongLong(x)) +#endif } } else { #if CYTHON_USE_PYLONG_INTERNALS const digit* digits = ((PyLongObject*)x)->ob_digit; switch (Py_SIZE(x)) { case 0: return (long) 0; - case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) -digits[0]) + case -1: __PYX_VERIFY_RETURN_INT(long, sdigit, (sdigit) (-(sdigit)digits[0])) case 1: __PYX_VERIFY_RETURN_INT(long, digit, +digits[0]) case -2: if (8 * sizeof(long) - 1 > 1 * PyLong_SHIFT) { @@ -11198,8 +12253,10 @@ #endif if (sizeof(long) <= sizeof(long)) { __PYX_VERIFY_RETURN_INT_EXC(long, long, PyLong_AsLong(x)) +#ifdef HAVE_LONG_LONG } else if (sizeof(long) <= sizeof(PY_LONG_LONG)) { __PYX_VERIFY_RETURN_INT_EXC(long, PY_LONG_LONG, PyLong_AsLongLong(x)) +#endif } } { @@ -11247,7 +12304,8 @@ return (long) -1; } -static int __Pyx_check_binary_version(void) { +/* CheckBinaryVersion */ + static int __Pyx_check_binary_version(void) { char ctversion[4], rtversion[4]; PyOS_snprintf(ctversion, 4, "%d.%d", PY_MAJOR_VERSION, PY_MINOR_VERSION); PyOS_snprintf(rtversion, 4, "%s", Py_GetVersion()); @@ -11262,7 +12320,8 @@ return 0; } -#ifndef __PYX_HAVE_RT_ImportModule +/* ModuleImport */ + #ifndef __PYX_HAVE_RT_ImportModule #define __PYX_HAVE_RT_ImportModule static PyObject *__Pyx_ImportModule(const char *name) { PyObject *py_name = 0; @@ -11279,7 +12338,8 @@ } #endif -#ifndef __PYX_HAVE_RT_ImportType +/* TypeImport */ + #ifndef __PYX_HAVE_RT_ImportType #define __PYX_HAVE_RT_ImportType static PyTypeObject *__Pyx_ImportType(const char *module_name, const char *class_name, size_t size, int strict) @@ -11343,7 +12403,8 @@ } #endif -static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { +/* InitStrings */ + static int __Pyx_InitStrings(__Pyx_StringTabEntry *t) { while (t->p) { #if PY_MAJOR_VERSION < 3 if (t->is_unicode) { @@ -11444,7 +12505,9 @@ else return PyObject_IsTrue(x); } static CYTHON_INLINE PyObject* __Pyx_PyNumber_IntOrLong(PyObject* x) { +#if CYTHON_USE_TYPE_SLOTS PyNumberMethods *m; +#endif const char *name = NULL; PyObject *res = NULL; #if PY_MAJOR_VERSION < 3 @@ -11453,8 +12516,9 @@ if (PyLong_Check(x)) #endif return __Pyx_NewRef(x); +#if CYTHON_USE_TYPE_SLOTS m = Py_TYPE(x)->tp_as_number; -#if PY_MAJOR_VERSION < 3 + #if PY_MAJOR_VERSION < 3 if (m && m->nb_int) { name = "int"; res = PyNumber_Int(x); @@ -11463,11 +12527,14 @@ name = "long"; res = PyNumber_Long(x); } -#else + #else if (m && m->nb_int) { name = "int"; res = PyNumber_Long(x); } + #endif +#else + res = PyNumber_Int(x); #endif if (res) { #if PY_MAJOR_VERSION < 3 diff -Nru s3ql-2.22+dfsg/src/s3ql/__init__.py s3ql-2.23+dfsg/src/s3ql/__init__.py --- s3ql-2.22+dfsg/src/s3ql/__init__.py 2017-06-23 16:36:06.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/__init__.py 2017-08-18 18:11:19.000000000 +0000 @@ -22,7 +22,7 @@ 'REV_VER_MAP', 'RELEASE', 'BUFSIZE', 'CTRL_NAME', 'CTRL_INODE' ] -VERSION = '2.22' +VERSION = '2.23' RELEASE = '%s' % VERSION # TODO: On next revision bump, consider removing support for TIME diff -Nru s3ql-2.22+dfsg/src/s3ql/mount.py s3ql-2.23+dfsg/src/s3ql/mount.py --- s3ql-2.22+dfsg/src/s3ql/mount.py 2017-03-23 17:00:17.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql/mount.py 2017-08-18 17:46:35.000000000 +0000 @@ -158,6 +158,7 @@ operations = fs.Operations(block_cache, db, max_obj_size=param['max_obj_size'], inode_cache=InodeCache(db, param['inode_gen']), upload_event=metadata_upload_thread.event) + block_cache.fs = operations metadata_upload_thread.fs = operations with ExitStack() as cm: diff -Nru s3ql-2.22+dfsg/src/s3ql.egg-info/PKG-INFO s3ql-2.23+dfsg/src/s3ql.egg-info/PKG-INFO --- s3ql-2.22+dfsg/src/s3ql.egg-info/PKG-INFO 2017-06-23 16:36:56.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql.egg-info/PKG-INFO 2017-08-18 18:11:57.000000000 +0000 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: s3ql -Version: 2.22 +Version: 2.23 Summary: a full-featured file system for online data storage Home-page: https://bitbucket.org/nikratio/s3ql/ Author: Nikolaus Rath @@ -146,6 +146,12 @@ Please report any bugs you may encounter in the `Bitbucket Issue Tracker`_. + Professional Support + ==================== + + Professional support is available. Please contact Nikolaus Rath + for details. + Contributing ============ diff -Nru s3ql-2.22+dfsg/src/s3ql.egg-info/requires.txt s3ql-2.23+dfsg/src/s3ql.egg-info/requires.txt --- s3ql-2.22+dfsg/src/s3ql.egg-info/requires.txt 2017-06-23 16:36:56.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql.egg-info/requires.txt 2017-08-18 18:11:57.000000000 +0000 @@ -1,6 +1,6 @@ apsw >= 3.7.0 -pycrypto -requests defusedxml dugong >= 3.4, < 4.0 llfuse >= 1.0, < 2.0 +pycrypto +requests diff -Nru s3ql-2.22+dfsg/src/s3ql.egg-info/SOURCES.txt s3ql-2.23+dfsg/src/s3ql.egg-info/SOURCES.txt --- s3ql-2.22+dfsg/src/s3ql.egg-info/SOURCES.txt 2017-06-23 16:36:59.000000000 +0000 +++ s3ql-2.23+dfsg/src/s3ql.egg-info/SOURCES.txt 2017-08-18 18:11:57.000000000 +0000 @@ -3,6 +3,18 @@ README.rst setup.cfg setup.py +/home/nikratio/in-progress/s3ql/doc/man/fsck.s3ql.1 +/home/nikratio/in-progress/s3ql/doc/man/mkfs.s3ql.1 +/home/nikratio/in-progress/s3ql/doc/man/mount.s3ql.1 +/home/nikratio/in-progress/s3ql/doc/man/s3ql_oauth_client.1 +/home/nikratio/in-progress/s3ql/doc/man/s3ql_verify.1 +/home/nikratio/in-progress/s3ql/doc/man/s3qladm.1 +/home/nikratio/in-progress/s3ql/doc/man/s3qlcp.1 +/home/nikratio/in-progress/s3ql/doc/man/s3qlctrl.1 +/home/nikratio/in-progress/s3ql/doc/man/s3qllock.1 +/home/nikratio/in-progress/s3ql/doc/man/s3qlrm.1 +/home/nikratio/in-progress/s3ql/doc/man/s3qlstat.1 +/home/nikratio/in-progress/s3ql/doc/man/umount.s3ql.1 bin/fsck.s3ql bin/mkfs.s3ql bin/mount.s3ql @@ -64,6 +76,9 @@ doc/html/_sources/special.txt doc/html/_sources/tips.txt doc/html/_sources/umount.txt +doc/html/_sources/include/about.txt +doc/html/_sources/include/exitcodes.txt +doc/html/_sources/include/postman.txt doc/html/_sources/man/adm.txt doc/html/_sources/man/cp.txt doc/html/_sources/man/ctrl.txt @@ -100,6 +115,9 @@ doc/html/_static/up-pressed.png doc/html/_static/up.png doc/html/_static/websupport.js +doc/html/include/about.html +doc/html/include/exitcodes.html +doc/html/include/postman.html doc/html/man/adm.html doc/html/man/cp.html doc/html/man/ctrl.html diff -Nru s3ql-2.22+dfsg/tests/t2_block_cache.py s3ql-2.23+dfsg/tests/t2_block_cache.py --- s3ql-2.22+dfsg/tests/t2_block_cache.py 2016-08-08 23:51:01.000000000 +0000 +++ s3ql-2.23+dfsg/tests/t2_block_cache.py 2017-08-18 17:46:35.000000000 +0000 @@ -231,7 +231,7 @@ # We want to expire 4 entries, 2 of which are already flushed self.cache.cache.max_entries = 16 - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=2) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=2) self.cache.expire() self.cache.backend_pool.verify() self.assertEqual(len(self.cache.cache), 16) @@ -254,7 +254,7 @@ data3 = self.random_data(datalen) # Case 1: create new object - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data1) @@ -263,7 +263,7 @@ self.cache.backend_pool.verify() # Case 2: Link new object - self.cache.backend_pool = TestBackendPool(self.backend_pool) + self.cache.backend_pool = MockBackendPool(self.backend_pool) with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data1) @@ -272,7 +272,7 @@ self.cache.backend_pool.verify() # Case 3: Upload old object, still has references - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data2) @@ -280,7 +280,7 @@ self.cache.backend_pool.verify() # Case 4: Upload old object, no references left - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_del=1, no_write=1) with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data3) @@ -288,7 +288,7 @@ self.cache.backend_pool.verify() # Case 5: Link old object, no references left - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_del=1) with self.cache.get(inode, blockno2) as fh: fh.seek(0) fh.write(data2) @@ -297,7 +297,7 @@ # Case 6: Link old object, still has references # (Need to create another object first) - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) with self.cache.get(inode, blockno3) as fh: fh.seek(0) fh.write(data1) @@ -305,7 +305,7 @@ self.cache.upload(el3) self.cache.backend_pool.verify() - self.cache.backend_pool = TestBackendPool(self.backend_pool) + self.cache.backend_pool = MockBackendPool(self.backend_pool) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data1) @@ -320,7 +320,7 @@ blockno2 = 24 data = self.random_data(datalen) - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) with self.cache.get(inode, blockno1) as fh: fh.seek(0) fh.write(data) @@ -330,7 +330,7 @@ self.cache.clear() self.cache.backend_pool.verify() - self.cache.backend_pool = TestBackendPool(self.backend_pool) + self.cache.backend_pool = MockBackendPool(self.backend_pool) self.cache.remove(inode, blockno1) self.cache.backend_pool.verify() @@ -450,10 +450,10 @@ with self.cache.get(inode, 1) as fh: fh.seek(0) fh.write(data1) - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) commit(self.cache, inode) self.cache.backend_pool.verify() - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_del=1) self.cache.remove(inode, 1) self.cache.backend_pool.verify() @@ -470,10 +470,10 @@ with self.cache.get(inode, 1) as fh: fh.seek(0) fh.write(data1) - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_write=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_write=1) self.cache.clear() self.cache.backend_pool.verify() - self.cache.backend_pool = TestBackendPool(self.backend_pool, no_del=1) + self.cache.backend_pool = MockBackendPool(self.backend_pool, no_del=1) self.cache.remove(inode, 1) self.cache.backend_pool.verify() with self.cache.get(inode, 1) as fh: @@ -481,7 +481,7 @@ self.assertEqual(fh.read(42), b'') -class TestBackendPool(AbstractBackend): +class MockBackendPool(AbstractBackend): has_native_rename = False def __init__(self, backend_pool, no_read=0, no_write=0, no_del=0):