diff -Nru nncp-8.5.0/bin/cmd.list nncp-8.7.2/bin/cmd.list --- nncp-8.5.0/bin/cmd.list 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/bin/cmd.list 2022-03-13 14:33:13.000000000 +0000 @@ -1,3 +1,4 @@ +nncp-ack nncp-bundle nncp-call nncp-caller diff -Nru nncp-8.5.0/debian/changelog nncp-8.7.2/debian/changelog --- nncp-8.5.0/debian/changelog 2022-02-16 14:33:23.000000000 +0000 +++ nncp-8.7.2/debian/changelog 2022-03-21 12:37:06.000000000 +0000 @@ -1,3 +1,28 @@ +nncp (8.7.2-1) unstable; urgency=medium + + * New upstream release. + * Update cron-daily example to use new nncp-rm capability. + + -- John Goerzen Mon, 21 Mar 2022 07:37:06 -0500 + +nncp (8.7.1-2) unstable; urgency=medium + + * Increase test timeout to 60 minutes for slow mipsel autobuilder. + + -- John Goerzen Sat, 12 Mar 2022 08:13:48 -0600 + +nncp (8.7.1-1) unstable; urgency=medium + + * New upstream release. + + -- John Goerzen Thu, 10 Mar 2022 10:06:54 -0600 + +nncp (8.7.0-1) unstable; urgency=medium + + * New upstream release, adds nncp-ack command. + + -- John Goerzen Fri, 04 Mar 2022 18:41:26 -0600 + nncp (8.5.0-1) unstable; urgency=medium * New upstream release. Features integrated Yggdrasil node. diff -Nru nncp-8.5.0/debian/control nncp-8.7.2/debian/control --- nncp-8.5.0/debian/control 2022-02-16 14:33:23.000000000 +0000 +++ nncp-8.7.2/debian/control 2022-03-09 13:50:05.000000000 +0000 @@ -6,7 +6,7 @@ Priority: optional Build-Depends: debhelper-compat (= 13), pandoc, texinfo, plantuml, dh-golang, - golang-go, + golang-go (>= 2:1.17~0), golang-github-davecgh-go-xdr-dev, golang-github-dustin-go-humanize-dev, golang-github-fsnotify-fsnotify-dev, diff -Nru nncp-8.5.0/debian/examples/cron-daily-nncp nncp-8.7.2/debian/examples/cron-daily-nncp --- nncp-8.5.0/debian/examples/cron-daily-nncp 2021-09-05 04:16:25.000000000 +0000 +++ nncp-8.7.2/debian/examples/cron-daily-nncp 2022-03-21 12:37:06.000000000 +0000 @@ -1,4 +1,8 @@ #!/bin/bash -find /var/spool/nncp -name '*.seen' -a -mtime +7 -delete -find /var/spool/nncp -name '*.hdr' -a -mtime +7 -delete +for TYPE in part seen hdr area; do + su nncp -s /bin/bash -c "nncp-rm -quiet -all -older 7d -$TYPE" +done + +su nncp -s /bin/bash -c "nncp-rm -quiet -tmp -older 7d" + diff -Nru nncp-8.5.0/debian/rules nncp-8.7.2/debian/rules --- nncp-8.5.0/debian/rules 2022-02-16 14:33:23.000000000 +0000 +++ nncp-8.7.2/debian/rules 2022-03-12 14:12:56.000000000 +0000 @@ -51,5 +51,5 @@ # Build was failing on 32-bit arm, timing out. override_dh_auto_test: - dh_auto_test -- -timeout 1800s + dh_auto_test -- -timeout 3600s diff -Nru nncp-8.5.0/doc/about.texi nncp-8.7.2/doc/about.texi --- nncp-8.5.0/doc/about.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/about.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,8 +1,18 @@ +@cindex about @strong{NNCP} (Node to Node copy) is a collection of utilities simplifying secure store-and-forward files, mail and command exchanging. See also this page @ref{Об утилитах, on russian}. +@cindex F2F +@cindex friend-to-friend +@cindex E2E +@cindex end-to-end +@cindex darknet +@cindex DTN +@cindex delay tolerant +@cindex dead drop +@cindex onion encryption This utilities are intended to help build up small size (dozens of nodes) ad-hoc @url{https://en.wikipedia.org/wiki/Friend-to-friend, friend-to-friend} (F2F) statically routed @@ -28,6 +38,9 @@ Look for possible @ref{Use cases, use cases}! +@cindex GPL +@cindex free software +@cindex licence NNCP is @url{https://www.gnu.org/philosophy/pragmatic.html, copylefted} @url{https://www.gnu.org/philosophy/free-sw.html, free software} licenced under @url{https://www.gnu.org/licenses/gpl-3.0.html, GNU GPLv3}. diff -Nru nncp-8.5.0/doc/admin.texi nncp-8.7.2/doc/admin.texi --- nncp-8.5.0/doc/admin.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/admin.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node Administration +@cindex administration @unnumbered Administration NNCP uses following files/directories you should be aware of: @@ -22,13 +23,16 @@ It can contain @file{.nock} files: fully downloaded, but still not checksummed. Can be checksummed (with @file{.nock} extension removing) - with @command{nncp-check -nock}. + with @command{@ref{nncp-check} -nock}. Also it can contain @file{seen/} and @file{hdr/} subdirectories, that should be cleaned too from time to time. - All of that cleaning tasks can be done with @ref{nncp-rm} utility. + All of that cleaning tasks can be done with @command{@ref{nncp-rm}} utility. + @cindex shared spool + @cindex setgid + @pindex umask @anchor{Shared spool} If you want to share single spool directory with multiple grouped Unix users, then you can @command{setgid} it and assure that umask @@ -47,6 +51,7 @@ @ref{Log} file, for example @file{/var/spool/nncp/log}. It should be rotated. Choose you own preferable way to do it. + @pindex newsyslog Example @url{https://www.newsyslog.org/manual.html, newsyslog}'s entry: @example /var/spool/nncp/log 644 7 100 * BCYN @@ -54,17 +59,21 @@ @item Optional @ref{CfgIncoming, incoming} directories where uploaded - files are stored. Probably you want to run @ref{nncp-reass} from - time to time to reassemble all chunked uploads. Example crontab + files are stored. Probably you want to run @command{@ref{nncp-reass}} + from time to time to reassemble all chunked uploads. Example crontab entry: @example */1 * * * * nncp-reass -all -noprogress @end example -@item - Possibly long running @ref{nncp-daemon}, @ref{nncp-caller}, - @ref{nncp-toss}, @ref{nncp-check} daemons. As all software, they can +@pindex daemontools +@pindex supervise +@pindex multilog +@item + Possibly long running @command{@ref{nncp-daemon}}, + @command{@ref{nncp-caller}}, @command{@ref{nncp-toss}}, + @command{@ref{nncp-check}} daemons. As all software, they can fail and you should place them under some supervisor control. For example you can use @url{http://cr.yp.to/daemontools.html, @@ -90,14 +99,17 @@ # mv /var/service/.nncp-toss /var/service/nncp-toss @end example +@pindex inetd @item - @ref{nncp-daemon} can also be run as + @command{@ref{nncp-daemon}} can also be run as @url{https://en.wikipedia.org/wiki/Inetd, inetd} service on UUCP's port: @example uucp stream tcp6 nowait nncpuser /usr/local/bin/nncp-daemon nncp-daemon -quiet -ucspi @end example +@cindex UCSPI +@pindex tcpserver @item Or it can be also run as a @command{daemontools} daemon under @url{http://cr.yp.to/ucspi-tcp.html, UCSPI-TCP}. In the example diff -Nru nncp-8.5.0/doc/building.texi nncp-8.7.2/doc/building.texi --- nncp-8.5.0/doc/building.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/building.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node Build-instructions +@cindex building @section Build instructions Make sure that Go is installed. For example to install it from packages: @@ -19,12 +20,19 @@ $ redo all @end example +@pindex info After that you should get various @command{bin/nncp-*} binaries and @command{bin/hjson-cli} command (only for your convenience, not necessary installation). For example, documentation for @command{nncp-bundle} command can be get with @command{info doc/nncp.info -n nncp-bundle}. +@pindex redo +@pindex apenwarr/redo +@pindex apenwarr/do +@pindex redo-c +@pindex baredo +@pindex goredo It uses @url{http://cr.yp.to/redo.html, redo} build system for that examples. You can use one of its various implementations, or at least minimalistic POSIX shell @command{contrib/do} (just replace @@ -35,13 +43,46 @@ from that project), @url{https://github.com/leahneukirchen/redo-c, redo-c}, @url{https://github.com/gotroyb127/baredo, baredo}. -There is @command{install} target respecting @env{$DESTDIR}. It will -install binaries and info-documentation: +@vindex PREFIX +@vindex DESTDIR +@vindex GO +@vindex MAKEINFO +@vindex PLANTUML +@vindex PREFIX +@vindex SENDMAIL +@vindex CFGPATH +@vindex SPOOLPATH +@vindex LOGPATH +@vindex BINDIR +@vindex INFODIR +@vindex DOCDIR +@file{config} file contains some environment variables that are +respected during installation: +@env{$PREFIX}, +@env{$DESTDIR}, +@env{$GO}, +@env{$MAKEINFO}, +@env{$PLANTUML}, +@env{$PREFIX}, +@env{$SENDMAIL}, +@env{$CFGPATH}, +@env{$SPOOLPATH}, +@env{$LOGPATH}, +@env{$BINDIR}, +@env{$INFODIR}, +@env{$DOCDIR}. + +There is @command{install} target for binaries and info-documentation +installation: @example # PREFIX=/usr/local redo install @end example +@vindex nofsnotify +@cindex kqueue +@cindex epoll +@vindex GO_CFLAGS NNCP depends on @code{github.com/fsnotify/fsnotify} library, that is solely relies on OS-specific mechanisms. There is possibility that you have either broken or unsupported ones. You can still build NNCP with @@ -49,7 +90,8 @@ usage at all: @example -$ GO_CFLAGS="-tags nofsnotify" redo ... +$ GO_CFLAGS="-tags nofsnotify" redo @dots{} @end example +@vindex noyggdrasil You can also disable Yggdrasil support with @code{-tags noyggdrasil}. diff -Nru nncp-8.5.0/doc/bundles.texi nncp-8.7.2/doc/bundles.texi --- nncp-8.5.0/doc/bundles.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/bundles.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,17 +1,20 @@ @node Bundles +@cindex bundles +@cindex tapes +@cindex streaming media @unnumbered Bundles -Usual @ref{nncp-xfer} command requires filesystem it can operate on. -That presumes random access media storage usage, like hard drives, USB -flash drives and similar. But media like CD-ROM and especially tape -drives are sequential by nature. You can prepare intermediate directory -for recording to CD-ROM disc/tape, but that requires additional storage -and is inconvenient. - -Bundles, created with @ref{nncp-bundle} command are convenient -alternative to ordinary @command{nncp-xfer}. Bundle is just a collection -of @ref{Encrypted, encrypted packets}, stream of packets. It could be -sequentially streamed for recording and digested back. +Usual @command{@ref{nncp-xfer}} command requires filesystem it can +operate on. That presumes random access media storage usage, like hard +drives, USB flash drives and similar. But media like CD-ROM and +especially tape drives are sequential by nature. You can prepare +intermediate directory for recording to CD-ROM disc/tape, but that +requires additional storage and is inconvenient. + +Bundles, created with @command{@ref{nncp-bundle}} command are convenient +alternative to ordinary @command{@ref{nncp-xfer}}. Bundle is just a +collection of @ref{Encrypted, encrypted packets}, stream of packets. It +could be sequentially streamed for recording and digested back. @itemize @@ -48,8 +51,11 @@ @end itemize +@pindex pax +@pindex tar Technically bundle is valid POSIX.1-2001 @url{https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_01, pax archive} with directory/files hierarchy identical to that is used in -@ref{nncp-xfer}: @file{NNCP/RECIPIENT/SENDER/PACKET}. So bundle can also -be created by manual tar-ing of @command{nncp-xfer} resulting directory. +@command{@ref{nncp-xfer}}: @file{NNCP/RECIPIENT/SENDER/PACKET}. +So bundle can also be created by manual tar-ing of +@command{@ref{nncp-xfer}} resulting directory. diff -Nru nncp-8.5.0/doc/call.texi nncp-8.7.2/doc/call.texi --- nncp-8.5.0/doc/call.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/call.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,7 +1,9 @@ @node Call +@cindex call +@vindex calls @unnumbered Call configuration -Call is a rule when and how node can be called by @ref{nncp-caller}. +Call is a rule when and how node can be called by @command{@ref{nncp-caller}}. Example list of call structures: @@ -51,17 +53,23 @@ @table @emph +@vindex cron @item cron @include cronexpr.texi +@vindex nice @item nice Optional. Use that @ref{Niceness, niceness} during the call (255 is used otherwise). +@vindex xx +@vindex rx +@vindex tx @item xx Optional. Either @verb{|rx|} or @verb{|tx|}. Tells only to either to receive or to transmit data during that call. +@vindex addr @item addr Optional. Call only that address, instead of trying all from @ref{CfgAddrs, @emph{addrs}} configuration option. It can be either key @@ -79,11 +87,13 @@ Optional. Override @ref{CfgMaxOnlineTime, @emph{maxonlinetime}} configuration option when calling. +@vindex autotoss @item autotoss, -doseen, -nofile, -nofreq, -noexec, -notrns Optionally enable auto tossing: run tosser on node's spool every second during the call. You can control either are @file{seen/} files must be created, or skip any kind of packet processing. +@vindex when-tx-exists @item when-tx-exists Call only if packets for sending exists. The check of outbound packets existence is performed @strong{every} time we are going to make a call, @@ -91,16 +101,19 @@ @emph{cron} configuration decides that it is time to make a call, with @emph{when-tx-exists} option it checks packets existence first. +@vindex nock @anchor{CfgNoCK} @item nock NoCK (no-checksumming) tells not to do checksumming of received files, -assuming that it will be done for example with @ref{nncp-check} command -later. That can help minimizing time spent online, because HDD won't do -simultaneous reading of the data for checksumming and writing of the -received one, but just sequential writing of the file. Pay attention -that you have to make a call to remote node after checksumming is done, -to send notification about successful packet reception. +assuming that it will be done for example with +@command{@ref{nncp-check}} command later. That can help minimizing time +spent online, because HDD won't do simultaneous reading of the data for +checksumming and writing of the received one, but just sequential +writing of the file. Pay attention that you have to make a call to +remote node after checksumming is done, to send notification about +successful packet reception. +@vindex mcd-ignore @anchor{CfgMCDIgnore} @item mcd-ignore Ignore @ref{MCD} announcements: do not add MCD addresses for possible diff -Nru nncp-8.5.0/doc/cfg/areas.texi nncp-8.7.2/doc/cfg/areas.texi --- nncp-8.5.0/doc/cfg/areas.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/areas.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node CfgAreas +@vindex areas @section Configuration areas options @ref{Multicast} areas configuration only used with multicast packets. @@ -36,6 +37,7 @@ The only required field is the @code{id}. You can not process multicast packets that has unknown area identification. +@vindex subs @code{subs} contains a list of recipients you must relay incoming multicast packet on. diff -Nru nncp-8.5.0/doc/cfg/dir.texi nncp-8.7.2/doc/cfg/dir.texi --- nncp-8.5.0/doc/cfg/dir.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/dir.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,9 +1,10 @@ @node Configuration directory +@cindex configuration directory @section Configuration directory Optionally you can convert configuration file to the directory layout -with @ref{nncp-cfgdir} command. And vice versa too, of course loosing -the comment lines. Directory layout can looks like that: +with @command{@ref{nncp-cfgdir}} command. And vice versa too, of course +loosing the comment lines. Directory layout can looks like that: @example nncp-cfg-dir @@ -95,8 +96,9 @@ Your @option{-cfg} and @env{$NNCPCFG} could point to that directory, instead of @file{.hjson} file. It will be transparently converted to internal JSON representation. However it can not be encrypted with the -@ref{nncp-cfgenc}. +@command{@ref{nncp-cfgenc}}. +@cindex private keys That layout should be much more machine friendly and scriptable. Each string parameters is stored as a single line plain text file. String arrays are newline-separated plain text files. Dictionaries are diff -Nru nncp-8.5.0/doc/cfg/general.texi nncp-8.7.2/doc/cfg/general.texi --- nncp-8.5.0/doc/cfg/general.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/general.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node CfgGeneral +@cindex general configuration options @section Configuration general options Those options are in the root of configuration dictionary. @@ -26,8 +27,13 @@ @end verbatim @table @code + +@vindex spool @item spool Absolute path to the @ref{Spool, spool} directory. + +@vindex log +@vindex FD log file descriptor @item log Either: @itemize @@ -35,36 +41,49 @@ @item @code{FD:XXX}, where @code{XXX} is a decimal file descriptor to write records too @end itemize + +@vindex umask @item umask Will force all invoked commands to override their umask to specified octal mask. Useful for using with @ref{Shared spool, shared spool directories}. + +@vindex noprogress @item noprogress When enabled, disables progress showing for many commands by default. You can always force its showing with @option{-progress} command line option anyway. + +@vindex nohdr @anchor{CfgNoHdr} @item nohdr @strong{nohdr} option disables @ref{HdrFile, @file{hdr/}} files usage. + @end table And optional @ref{MCD, MultiCast Discovery} options: @table @code + +@vindex mcd-listen @anchor{CfgMCDListen} @item mcd-listen Specifies list of network interfaces regular expression -@ref{nncp-caller} will listen for incoming @ref{MCD} announcements. +@command{@ref{nncp-caller}} will listen for incoming @ref{MCD} announcements. + +@vindex mcd-send @anchor{CfgMCDSend} @item mcd-send Specifies list of network interfaces regular expressions, and intervals -in seconds, where @ref{nncp-daemon} will send @ref{MCD} announcements. +in seconds, where @command{@ref{nncp-daemon}} will send @ref{MCD} announcements. + @end table +@cindex yggdrasil aliases @anchor{CfgYggdrasilAliases} Optional @ref{Yggdrasil}-related aliases are used for convenience and keeping private keys away being used directly in command line. Each @code{PUB}, @code{PRV}, @code{PEER}, @code{BIND} value in -@ref{nncp-daemon}'s @option{-yggdrasil} and in @code{yggdrasil:} +@command{@ref{nncp-daemon}}'s @option{-yggdrasil} and in @code{yggdrasil:} addresses is replaced with alias value. Moreover each entry in list of @code{PUB}s, @code{PEER}s and @code{BIND} can be an alias too. Pay attention, that all aliases ending with @code{prv} will be saved with diff -Nru nncp-8.5.0/doc/cfg/index.texi nncp-8.7.2/doc/cfg/index.texi --- nncp-8.5.0/doc/cfg/index.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/index.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,9 +1,11 @@ @node Configuration +@cindex configuration file @unnumbered Configuration file +@cindex Hjson NNCP uses single file configuration file in @url{https://hjson.org/, Hjson} format (see also section about @ref{Configuration directory, -directory layout}) . Initially it is created with @ref{nncp-cfgnew} +directory layout}) . Initially it is created with @command{@ref{nncp-cfgnew}} command and at minimum it can look like this: @verbatim @@ -30,6 +32,10 @@ } @end verbatim +@cindex JSON +@pindex hjson-cli +@pindex gojq +@pindex gjo Do not forget that Hjson can be safely converted to JSON and vice versa (loosing formatting and comments of course). By default @command{hjson-cli} utility from @code{github.com/hjson/hjson-go} is diff -Nru nncp-8.5.0/doc/cfg/neigh.texi nncp-8.7.2/doc/cfg/neigh.texi --- nncp-8.5.0/doc/cfg/neigh.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/neigh.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,6 +1,8 @@ @node CfgNeigh +@cindex neighbour configuration options @section Configuration neighbour options +@vindex neigh @strong{neigh} section contains all known neighbours information. It always has @strong{self} neighbour that is copy of our node's public data (public keys). It is useful for copy-paste sharing with your @@ -65,8 +67,10 @@ If present, then node can be online called using @ref{Sync, synchronization protocol}. Contains authentication public key. +@vindex exec +@pindex sendmail @anchor{CfgExec} - @item exec +@item exec Dictionary consisting of handles and corresponding command line arguments. In example above there are @command{sendmail} handles, @command{warcer}, @command{wgeter} and @command{flag} one. Remote @@ -87,11 +91,13 @@ feeding @verb{|hello world\n|} to that started @command{sendmail} process. +@vindex incoming @anchor{CfgIncoming} @item incoming Full path to directory where all file uploads will be saved. May be omitted to forbid file uploading on that node. +@vindex freq @anchor{CfgFreq} @item freq @table @code @@ -108,6 +114,7 @@ transmission. @end table +@vindex via @anchor{CfgVia} @item via An array of node identifiers that will be used as a relay to that @@ -116,11 +123,12 @@ nodes. May be omitted if direct connection exists and no relaying is required. +@vindex addrs @anchor{CfgAddrs} @item addrs Dictionary containing known network addresses of the node. Each key is human-readable name of the address. For direct TCP connections - use @verb{|host:port|} format, pointing to @ref{nncp-daemon}'s + use @verb{|host:port|} format, pointing to @command{@ref{nncp-daemon}}'s listening instance. Also you can pipe connection through the external command using @@ -128,12 +136,14 @@ start and its @code{stdin}/@code{stdout} used as a connection. To use @ref{Yggdrasil} network for connectivity, use - @verb{|yggdrasil:PUB;PRV;PEER[,...]|} format, read about + @code{yggdrasil:PUB;PRV;PEER[,@dots{}]} format, read about @ref{CfgYggdrasilAliases, possible aliases} usage. May be omitted if either no direct connection exists, or - @ref{nncp-call} is used with forced address specifying. + @command{@ref{nncp-call}} is used with forced address specifying. +@vindex rxrate +@vindex txrate @anchor{CfgXxRate} @item rxrate/txrate If greater than zero, then at most *rate packets per second will be @@ -141,6 +151,7 @@ bandwidth traffic shaper: each packet has at most 64 KiB payload size. If omitted -- no rate limits. +@vindex onlinedeadline @anchor{CfgOnlineDeadline} @item onlinedeadline Online connection deadline of nodes inactivity in seconds. It is the @@ -151,6 +162,7 @@ delays), wait for appearing packets ready to send and notifying remote side about their appearance. +@vindex maxonlinetime @anchor{CfgMaxOnlineTime} @item maxonlinetime If greater than zero, then it is maximal time of single connection. @@ -159,6 +171,6 @@ @anchor{CfgCalls} @item calls List of @ref{Call, call configuration}s. - Can be omitted if @ref{nncp-caller} won't be used to call that node. + Can be omitted if @command{@ref{nncp-caller}} won't be used to call that node. @end table diff -Nru nncp-8.5.0/doc/cfg/notify.texi nncp-8.7.2/doc/cfg/notify.texi --- nncp-8.5.0/doc/cfg/notify.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/notify.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,7 @@ @node CfgNotify +@cindex email notification +@cindex notification configuration options +@cindex logging handles @section Configuration notification options That section controls what notifications are enabled and how must be diff -Nru nncp-8.5.0/doc/cfg/self.texi nncp-8.7.2/doc/cfg/self.texi --- nncp-8.5.0/doc/cfg/self.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cfg/self.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,10 +1,18 @@ @node CfgSelf +@cindex self-node configuration keypairs @section Configuration self-node keypairs @strong{self} section contains our node's private keypairs. +@vindex ExchPrv +@vindex ExchPub +@vindex SignPrv +@vindex SignPub @strong{exch*} and @strong{sign*} are used during @ref{Encrypted, encrypted} packet creation. +@vindex NoisePrv +@vindex NoisePub @strong{noise*} are used during @ref{Sync, synchronization protocol} -working in @ref{nncp-call}, @ref{nncp-caller}, @ref{nncp-daemon}. +working in @command{@ref{nncp-call}}, @command{@ref{nncp-caller}}, +@command{@ref{nncp-daemon}}. diff -Nru nncp-8.5.0/doc/chunked.texi nncp-8.7.2/doc/chunked.texi --- nncp-8.5.0/doc/chunked.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/chunked.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node Chunked +@cindex chunked @unnumbered Chunked files There is ability to transfer huge files with dividing them into smaller @@ -10,9 +11,11 @@ storage devices, and/or at different time, reassembling the whole packet on the destination node. -Splitting is done with @ref{nncp-file, nncp-file -chunked} command and -reassembling with @ref{nncp-reass} command. +Splitting is done with @command{@ref{nncp-file} -chunked} command and +reassembling with @command{@ref{nncp-reass}} command. +@vindex .nncp.meta +@vindex .nncp.chunk Chunked @file{FILE} produces @file{FILE.nncp.meta}, @file{FILE.nncp.chunk0}, @file{FILE.nncp.chunk1}, @dots{} files. All @file{.nncp.chunkXXX} can be concatenated together to produce original @@ -44,6 +47,7 @@ @ref{MTH} checksum of each chunk @end multitable +@cindex ZFS recordsize @anchor{ChunkedZFS} It is strongly advisable to reassemble incoming chunked files on @url{https://en.wikipedia.org/wiki/ZFS, ZFS} dataset with deduplication diff -Nru nncp-8.5.0/doc/cmd/index.texi nncp-8.7.2/doc/cmd/index.texi --- nncp-8.5.0/doc/cmd/index.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/index.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,9 +1,11 @@ @node Commands +@cindex commands @unnumbered Commands Nearly all commands have the following common options: @table @option +@vindex NNCPCFG @item -cfg Path to configuration file. May be overridden by @env{$NNCPCFG} environment variable. If file file is an encrypted @ref{EBlob, @@ -24,16 +26,18 @@ Override @ref{CfgVia, via} configuration option for destination node. Specified nodes must be separated with comma: @verb{|NODE1,NODE2|}. With @verb{|-via -|} you can disable relaying at all. +@vindex NNCPSPOOL @item -spool Override path to spool directory. May be specified by @env{$NNCPSPOOL} environment variable. +@vindex NNCPLOG @item -log Override path to logfile. May be specified by @env{$NNCPLOG} environment variable. @item -quiet Print only errors, omit simple informational messages. In any case those messages are logged, so you can reread them using - @ref{nncp-log} command. + @command{@ref{nncp-log}} command. @item -progress, -noprogress Either force progress showing, or disable it. @item -version @@ -42,6 +46,15 @@ Print warranty information (no warranty). @end table +@vindex TMPDIR +All commands respect @env{$TMPDIR} environment variable. + +@vindex NNCPNOSYNC +If you set @env{$NNCPNOSYNC=1}, then all @code{fsync} operations will be +skipped. That is dangerous option, but if there there are abilities to +rollback possibly corrupted state to some stable snapshot, then disabled +@code{fsync} can give considerable increase in performance. + @menu Configuration file commands @@ -56,6 +69,7 @@ * nncp-exec:: * nncp-freq:: * nncp-trns:: +* nncp-ack:: Packets sharing commands @@ -92,6 +106,7 @@ @include cmd/nncp-exec.texi @include cmd/nncp-freq.texi @include cmd/nncp-trns.texi +@include cmd/nncp-ack.texi @include cmd/nncp-xfer.texi @include cmd/nncp-bundle.texi @include cmd/nncp-toss.texi diff -Nru nncp-8.5.0/doc/cmd/nncp-ack.texi nncp-8.7.2/doc/cmd/nncp-ack.texi --- nncp-8.5.0/doc/cmd/nncp-ack.texi 1970-01-01 00:00:00.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-ack.texi 2022-03-13 14:33:13.000000000 +0000 @@ -0,0 +1,78 @@ +@node nncp-ack +@cindex packet acknowledgement +@pindex nncp-ack +@section nncp-ack + +@example +$ nncp-ack [options] -all +$ nncp-ack [options] -node NODE[,@dots{}] +$ nncp-ack [options] -node NODE -pkt PKT + +$ nncp-ack [@dots{}] 4>&1 >&2 | nncp-rm [@dots{}] -pkt +@end example + +Send acknowledgement of successful @option{PKT} (Base32-encoded hash) +packet receipt from @option{NODE} node. If no @option{-pkt} is +specified, then acknowledge all packet in node's @code{rx} outbound +spool. If @option{-all} is specified, then do that for all nodes. + +That commands outputs list of created encrypted ACK packets +(@code{NODE/PKT}) to @strong{4}th file descriptor. That output can be +passed for example to @command{@ref{nncp-rm}} to remove them after +transmission to not wait for acknowledgement and retransmission. + +General workflow with acknowledgement is following, assuming that +Alice has some outbound packets for Bob: + +@itemize + +@item Transfer encrypted packets, without deleting them locally: + +@example +alice$ nncp-xfer -keep -tx -node bob /mnt/shared +@end example + +@item On Bob's side retrieve those packets: + +@example +bob$ nncp-xfer -rx /mnt/shared +@end example + +That will also check if copied packets checksum is not mismatched. + +@item Create ACK packets of received ones, saving the list of encrypted + ACK packets: + +@example +bob$ nncp-ack -node alice 4>acks +@end example + +@item Send those newly created packets back to Alice: + +@example +bob$ nncp-xfer [-keep] -tx /mnt/shared +@end example + +@item Remove them from outbound spool, because we expect no + acknowledgement for them: + +@example +bob$ nncp-rm -node alice -pkt ... -$ nncp-bundle [options] -rx -delete [-dryrun] [NODE @dots{}] < ... -$ nncp-bundle [options] -rx [-check] [-dryrun] [NODE @dots{}] < ... +$ nncp-bundle [options] -tx [-delete] NODE [NODE @dots{}] > @dots{} +$ nncp-bundle [options] -rx -delete [-dryrun] [NODE @dots{}] < @dots{} +$ nncp-bundle [options] -rx [-check] [-dryrun] [NODE @dots{}] < @dots{} @end example With @option{-tx} option, this command creates @ref{Bundles, bundle} of @@ -17,8 +18,8 @@ tape drives could easily provide too much throughput your CPU won't be able to verify on the fly. So if you won't @ref{nncp-toss, toss} received packets at the place, it is advisable either to run -@ref{nncp-check} utility for packets integrity verification, or to use -@option{-check} option to enable on the fly integrity check. +@command{@ref{nncp-check}} utility for packets integrity verification, +or to use @option{-check} option to enable on the fly integrity check. You can specify multiple @option{NODE} arguments, telling for what nodes you want to create the stream, or take it from. If no nodes are diff -Nru nncp-8.5.0/doc/cmd/nncp-caller.texi nncp-8.7.2/doc/cmd/nncp-caller.texi --- nncp-8.5.0/doc/cmd/nncp-caller.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-caller.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-caller +@pindex nncp-caller @section nncp-caller @example @@ -12,4 +13,4 @@ Otherwise all nodes with specified @emph{calls} configuration field will be called. -Look at @ref{nncp-call} for more information. +Look at @command{@ref{nncp-call}} for more information. diff -Nru nncp-8.5.0/doc/cmd/nncp-call.texi nncp-8.7.2/doc/cmd/nncp-call.texi --- nncp-8.5.0/doc/cmd/nncp-call.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-call.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-call +@pindex nncp-call @section nncp-call @example @@ -43,10 +44,10 @@ success. Optionally you can force @option{FORCEADDR} address usage, instead of addresses taken from configuration file. You can specify @verb{|host:port|}, @verb{#|some command#} and -@verb{|yggdrasil:PUB;PRV;PEER[,...]|} formats. +@code{yggdrasil:PUB;PRV;PEER[,@dots{}]} formats. If you specify @option{-ucspi} option, then it is assumed that you run -@command{nncp-call} command under some UCSPI-TCP compatible utility, +@command{@ref{nncp-call}} command under some UCSPI-TCP compatible utility, that provides read/write channels through 6/7 file descriptors. @option{-mcd-wait} options tells to wait up to specified number of @@ -55,7 +56,7 @@ @option{-autotoss} option runs tosser on node's spool every second during the call. All @option{-autotoss-*} options is the same as in -@ref{nncp-toss} command. +@command{@ref{nncp-toss}} command. Partly downloaded packets are stored in @file{.part} files. By default all downloaded files are sequentially checksummed in the background, @@ -64,8 +65,8 @@ files to @file{.nock} extension. Pay attention that checksumming can be time consuming and connection could be lost during that check, so remote node won't be notified that the file is finished. If you run -@ref{nncp-check, @command{nncp-check -nock}}, that will checksum files -and strip the @file{.nock} extension, then repeated call to remote node -will notify about packet's completion. Also it will be notified if -@ref{nncp-toss, tossing} created @file{seen/} file. -Read @ref{CfgNoCK, more} about @option{-nock} option. +@command{@ref{nncp-check} -nock}, that will checksum files and strip the +@file{.nock} extension, then repeated call to remote node will notify about +packet's completion. Also it will be notified if @ref{nncp-toss, tossing} +created @file{seen/} file. Read @ref{CfgNoCK, more} about @option{-nock} +option. diff -Nru nncp-8.5.0/doc/cmd/nncp-cfgdir.texi nncp-8.7.2/doc/cmd/nncp-cfgdir.texi --- nncp-8.5.0/doc/cmd/nncp-cfgdir.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-cfgdir.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-cfgdir +@pindex nncp-cfgdir @section nncp-cfgdir @example @@ -9,4 +10,4 @@ @option{-dump} option dumps current configuration file to the @ref{Configuration directory, directory layout} at @file{/path/to/dir}. @option{-load} loads it and parses, outputing the resulting Hjson to -stdout. +@code{stdout}. diff -Nru nncp-8.5.0/doc/cmd/nncp-cfgenc.texi nncp-8.7.2/doc/cmd/nncp-cfgenc.texi --- nncp-8.5.0/doc/cmd/nncp-cfgenc.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-cfgenc.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-cfgenc +@pindex nncp-cfgenc @section nncp-cfgenc @example diff -Nru nncp-8.5.0/doc/cmd/nncp-cfgmin.texi nncp-8.7.2/doc/cmd/nncp-cfgmin.texi --- nncp-8.5.0/doc/cmd/nncp-cfgmin.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-cfgmin.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,7 @@ @node nncp-cfgmin +@cindex stripped configuration +@cindex minimized configuration +@pindex nncp-cfgmin @section nncp-cfgmin @example @@ -7,5 +10,5 @@ Print out stripped configuration version: only path to @ref{Spool, spool}, path to log file, neighbours public keys are stayed. This is -useful mainly for usage with @ref{nncp-xfer} that has to know only -neighbours, without private keys involving. +useful mainly for usage with @command{@ref{nncp-xfer}} that has to know +only neighbours, without private keys involving. diff -Nru nncp-8.5.0/doc/cmd/nncp-cfgnew.texi nncp-8.7.2/doc/cmd/nncp-cfgnew.texi --- nncp-8.5.0/doc/cmd/nncp-cfgnew.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-cfgnew.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-cfgnew +@pindex nncp-cfgnew @section nncp-cfgnew @example diff -Nru nncp-8.5.0/doc/cmd/nncp-check.texi nncp-8.7.2/doc/cmd/nncp-check.texi --- nncp-8.5.0/doc/cmd/nncp-check.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-check.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-check +@pindex nncp-check @section nncp-check @example diff -Nru nncp-8.5.0/doc/cmd/nncp-cronexpr.texi nncp-8.7.2/doc/cmd/nncp-cronexpr.texi --- nncp-8.5.0/doc/cmd/nncp-cronexpr.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-cronexpr.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-cronexpr +@pindex nncp-cronexpr @section nncp-cronexpr @example diff -Nru nncp-8.5.0/doc/cmd/nncp-daemon.texi nncp-8.7.2/doc/cmd/nncp-daemon.texi --- nncp-8.5.0/doc/cmd/nncp-daemon.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-daemon.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-daemon +@pindex nncp-daemon @section nncp-daemon @example @@ -10,8 +11,8 @@ Start listening TCP daemon, wait for incoming connections and run @ref{Sync, synchronization protocol} with each of them. You can run -@ref{nncp-toss} utility in background to process inbound packets from -time to time. +@command{@ref{nncp-toss}} utility in background to process inbound +packets from time to time. @option{-maxconn} option specifies how many simultaneous clients daemon can handle. @option{-bind} option specifies @option{addr:port} it must @@ -36,7 +37,7 @@ @option{-autotoss} option runs tosser on node's spool every second during the call. All @option{-autotoss-*} options is the same as in -@ref{nncp-toss} command. +@command{@ref{nncp-toss}} command. Read @ref{CfgNoCK, more} about @option{-nock} option. diff -Nru nncp-8.5.0/doc/cmd/nncp-exec.texi nncp-8.7.2/doc/cmd/nncp-exec.texi --- nncp-8.5.0/doc/cmd/nncp-exec.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-exec.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-exec +@pindex nncp-exec @section nncp-exec @example @@ -22,6 +23,9 @@ } @end verbatim +@vindex NNCP_SELF +@vindex NNCP_SENDER +@vindex NNCP_NICE then executing @verb{|echo My message | nncp-exec -replynice 123 REMOTE sendmail root@localhost|} will lead to execution of: diff -Nru nncp-8.5.0/doc/cmd/nncp-file.texi nncp-8.7.2/doc/cmd/nncp-file.texi --- nncp-8.5.0/doc/cmd/nncp-file.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-file.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-file +@pindex nncp-file @section nncp-file @example diff -Nru nncp-8.5.0/doc/cmd/nncp-freq.texi nncp-8.7.2/doc/cmd/nncp-freq.texi --- nncp-8.5.0/doc/cmd/nncp-freq.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-freq.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-freq +@pindex nncp-freq @section nncp-freq @example diff -Nru nncp-8.5.0/doc/cmd/nncp-hash.texi nncp-8.7.2/doc/cmd/nncp-hash.texi --- nncp-8.5.0/doc/cmd/nncp-hash.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-hash.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,11 +1,12 @@ @node nncp-hash +@pindex nncp-hash @section nncp-hash @example $ nncp-hash [-file @dots{}] [-seek X] [-debug] [-progress] @end example -Calculate @ref{MTH} hash of either stdin, or @option{-file} if +Calculate @ref{MTH} hash of either @code{stdin}, or @option{-file} if specified. You can optionally force seeking the file first, reading only part of diff -Nru nncp-8.5.0/doc/cmd/nncp-log.texi nncp-8.7.2/doc/cmd/nncp-log.texi --- nncp-8.5.0/doc/cmd/nncp-log.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-log.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-log +@pindex nncp-log @section nncp-log @example diff -Nru nncp-8.5.0/doc/cmd/nncp-pkt.texi nncp-8.7.2/doc/cmd/nncp-pkt.texi --- nncp-8.5.0/doc/cmd/nncp-pkt.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-pkt.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-pkt +@pindex nncp-pkt @section nncp-pkt @example @@ -20,7 +21,7 @@ @option{-dump} option outputs plain packet's payload (if it is file transmission, then it will be the file itself as an example). If it is an encrypted packet, then it will be decrypted first, outputing the -included plain packet, that can be fed to @command{nncp-pkt} again: +included plain packet, that can be fed to @command{@ref{nncp-pkt}} again: @example Packet type: plain diff -Nru nncp-8.5.0/doc/cmd/nncp-reass.texi nncp-8.7.2/doc/cmd/nncp-reass.texi --- nncp-8.5.0/doc/cmd/nncp-reass.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-reass.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,4 +1,5 @@ @node nncp-reass +@pindex nncp-reass @section nncp-reass @example diff -Nru nncp-8.5.0/doc/cmd/nncp-rm.texi nncp-8.7.2/doc/cmd/nncp-rm.texi --- nncp-8.5.0/doc/cmd/nncp-rm.texi 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/doc/cmd/nncp-rm.texi 2022-03-13 14:33:13.000000000 +0000 @@ -1,43 +1,51 @@ @node nncp-rm +@pindex nncp-rm @section nncp-rm @example -$ nncp-rm [options] -tmp +$ nncp-rm [options] [-older X] -tmp $ nncp-rm [options] -lock -$ nncp-rm [options] @{-all|-node NODE@} -part -$ nncp-rm [options] @{-all|-node NODE@} -seen -$ nncp-rm [options] @{-all|-node NODE@} -nock -$ nncp-rm [options] @{-all|-node NODE@} -hdr -$ nncp-rm [options] @{-all|-node NODE@} -area -$ nncp-rm [options] @{-all|-node NODE@} [-rx] [-tx] -$ nncp-rm [options] @{-all|-node NODE@} -pkt PKT +$ nncp-rm [options] [-older X] @{-all|-node NODE@} -part +$ nncp-rm [options] [-older X] @{-all|-node NODE@} -seen +$ nncp-rm [options] [-older X] @{-all|-node NODE@} -nock +$ nncp-rm [options] [-older X] @{-all|-node NODE@} -area +$ nncp-rm [options] [-older X] @{-all|-node NODE@} @{-rx|-tx@} [-hdr] +$ nncp-rm [options] [-older X] @{-all|-node NODE@} -pkt < "$tarball".asc meta4-create -file "$tarball" -mtime "$tarball" -sig "$tarball".asc \ http://www.nncpgo.org/download/"$tarball" \ + http://y.www.nncpgo.org/download/"$tarball" \ https://nncp.mirrors.quux.org/download/"$tarball" > "$tarball".meta4 size=$(( $(stat -f %z $tarball) / 1024 )) diff -Nru nncp-8.5.0/ports/nncp/Makefile nncp-8.7.2/ports/nncp/Makefile --- nncp-8.5.0/ports/nncp/Makefile 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/ports/nncp/Makefile 2022-03-13 14:33:13.000000000 +0000 @@ -1,5 +1,5 @@ PORTNAME= nncp -DISTVERSION= 7.6.0 +DISTVERSION= 8.7.1 CATEGORIES= net MASTER_SITES= http://www.nncpgo.org/download/ diff -Nru nncp-8.5.0/ports/nncp/pkg-plist nncp-8.7.2/ports/nncp/pkg-plist --- nncp-8.5.0/ports/nncp/pkg-plist 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/ports/nncp/pkg-plist 2022-03-13 14:33:13.000000000 +0000 @@ -1,3 +1,4 @@ +bin/nncp-ack bin/nncp-bundle bin/nncp-call bin/nncp-caller diff -Nru nncp-8.5.0/src/call.go nncp-8.7.2/src/call.go --- nncp-8.5.0/src/call.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/call.go 2022-03-13 14:33:13.000000000 +0000 @@ -50,6 +50,7 @@ AutoTossNoExec bool AutoTossNoTrns bool AutoTossNoArea bool + AutoTossNoACK bool } func (ctx *Ctx) CallNode( @@ -116,7 +117,7 @@ ctx.LogI("call-started", les, func(les LEs) string { return fmt.Sprintf("Connection to %s (%s)", node.Name, addr) }) - state.Wait() + isGood = state.Wait() ctx.LogI("call-finished", append( les, LE{"Duration", int64(state.Duration.Seconds())}, @@ -137,7 +138,6 @@ humanize.IBytes(uint64(state.TxSpeed)), ) }) - isGood = true conn.Close() break } else { diff -Nru nncp-8.5.0/src/cfg.go nncp-8.7.2/src/cfg.go --- nncp-8.5.0/src/cfg.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cfg.go 2022-03-13 14:33:13.000000000 +0000 @@ -38,6 +38,7 @@ CfgPathEnv = "NNCPCFG" CfgSpoolEnv = "NNCPSPOOL" CfgLogEnv = "NNCPLOG" + CfgNoSync = "NNCPNOSYNC" ) var ( diff -Nru nncp-8.5.0/src/cmd/nncp-ack/main.go nncp-8.7.2/src/cmd/nncp-ack/main.go --- nncp-8.5.0/src/cmd/nncp-ack/main.go 1970-01-01 00:00:00.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-ack/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -0,0 +1,237 @@ +/* +NNCP -- Node to Node copy, utilities for store-and-forward data exchange +Copyright (C) 2016-2022 Sergey Matveev + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, version 3 of the License. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +*/ + +// Send packet receipt acknowledgement via NNCP. +package main + +import ( + "bufio" + "errors" + "flag" + "fmt" + "io" + "log" + "os" + "path/filepath" + "strings" + + xdr "github.com/davecgh/go-xdr/xdr2" + "go.cypherpunks.ru/nncp/v8" +) + +func usage() { + fmt.Fprintf(os.Stderr, nncp.UsageHeader()) + fmt.Fprintf(os.Stderr, "nncp-ack -- send packet receipt acknowledgement\n\n") + fmt.Fprintf(os.Stderr, "Usage: %s [options] -all\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] -node NODE[,...]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] -node NODE -pkt PKT\n", os.Args[0]) + fmt.Fprintln(os.Stderr, "Options:") + flag.PrintDefaults() +} + +func main() { + var ( + cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") + niceRaw = flag.String("nice", nncp.NicenessFmt(nncp.DefaultNiceFreq), "Outbound packet niceness") + minSizeRaw = flag.Uint64("minsize", 0, "Minimal required resulting packet size, in KiB") + viaOverride = flag.String("via", "", "Override Via path to destination node (ignored with -all)") + spoolPath = flag.String("spool", "", "Override path to spool") + logPath = flag.String("log", "", "Override path to logfile") + doAll = flag.Bool("all", false, "ACK all rx packet for all nodes") + nodesRaw = flag.String("node", "", "ACK rx packets for that node") + pktRaw = flag.String("pkt", "", "ACK only that packet") + quiet = flag.Bool("quiet", false, "Print only errors") + showPrgrs = flag.Bool("progress", false, "Force progress showing") + omitPrgrs = flag.Bool("noprogress", false, "Omit progress showing") + debug = flag.Bool("debug", false, "Print debug messages") + version = flag.Bool("version", false, "Print version information") + warranty = flag.Bool("warranty", false, "Print warranty information") + ) + log.SetFlags(log.Lshortfile) + flag.Usage = usage + flag.Parse() + if *warranty { + fmt.Println(nncp.Warranty) + return + } + if *version { + fmt.Println(nncp.VersionGet()) + return + } + nice, err := nncp.NicenessParse(*niceRaw) + if err != nil { + log.Fatalln(err) + } + + ctx, err := nncp.CtxFromCmdline( + *cfgPath, + *spoolPath, + *logPath, + *quiet, + *showPrgrs, + *omitPrgrs, + *debug, + ) + if err != nil { + log.Fatalln("Error during initialization:", err) + } + if ctx.Self == nil { + log.Fatalln("Config lacks private keys") + } + + ctx.Umask() + minSize := int64(*minSizeRaw) * 1024 + + var nodes []*nncp.Node + if *nodesRaw != "" { + for _, nodeRaw := range strings.Split(*nodesRaw, ",") { + node, err := ctx.FindNode(nodeRaw) + if err != nil { + log.Fatalln("Invalid -node specified:", err) + } + nodes = append(nodes, node) + } + } + if *doAll { + if len(nodes) != 0 { + usage() + os.Exit(1) + } + for _, node := range ctx.Neigh { + nodes = append(nodes, node) + } + } else if len(nodes) == 0 { + usage() + os.Exit(1) + } + + acksCreated := os.NewFile(uintptr(4), "ACKsCreated") + if acksCreated == nil { + log.Fatalln("can not open FD:4") + } + + if *pktRaw != "" { + if len(nodes) != 1 { + usage() + os.Exit(1) + } + nncp.ViaOverride(*viaOverride, ctx, nodes[0]) + pktName, err := ctx.TxACK(nodes[0], nice, *pktRaw, minSize) + if err != nil { + log.Fatalln(err) + } + acksCreated.WriteString(nodes[0].Id.String() + "/" + pktName + "\n") + return + } + + isBad := false + for _, node := range nodes { + for job := range ctx.Jobs(node.Id, nncp.TRx) { + pktName := filepath.Base(job.Path) + sender := ctx.Neigh[*job.PktEnc.Sender] + les := nncp.LEs{ + {K: "Node", V: job.PktEnc.Sender}, + {K: "Pkt", V: pktName}, + } + logMsg := func(les nncp.LEs) string { + return fmt.Sprintf( + "ACKing %s/%s", + ctx.NodeName(job.PktEnc.Sender), pktName, + ) + } + if sender == nil { + err := errors.New("unknown node") + ctx.LogE("ack-read", les, err, logMsg) + isBad = true + continue + } + fd, err := os.Open(job.Path) + if err != nil { + ctx.LogE("ack-read-open", les, err, func(les nncp.LEs) string { + return logMsg(les) + ": opening" + job.Path + }) + isBad = true + continue + } + pktEnc, _, err := ctx.HdrRead(fd) + if err != nil { + fd.Close() + ctx.LogE("ack-read-read", les, err, func(les nncp.LEs) string { + return logMsg(les) + ": reading" + job.Path + }) + isBad = true + continue + } + switch pktEnc.Magic { + case nncp.MagicNNCPEv1.B: + err = nncp.MagicNNCPEv1.TooOld() + case nncp.MagicNNCPEv2.B: + err = nncp.MagicNNCPEv2.TooOld() + case nncp.MagicNNCPEv3.B: + err = nncp.MagicNNCPEv3.TooOld() + case nncp.MagicNNCPEv4.B: + err = nncp.MagicNNCPEv4.TooOld() + case nncp.MagicNNCPEv5.B: + err = nncp.MagicNNCPEv5.TooOld() + case nncp.MagicNNCPEv6.B: + default: + err = errors.New("is not an encrypted packet") + } + if err != nil { + fd.Close() + ctx.LogE("ack-read-magic", les, err, logMsg) + isBad = true + continue + } + if _, err = fd.Seek(0, io.SeekStart); err != nil { + fd.Close() + ctx.LogE("ack-read-seek", les, err, func(les nncp.LEs) string { + return logMsg(les) + ": seeking" + }) + isBad = true + continue + } + pipeR, pipeW := io.Pipe() + go nncp.PktEncRead(ctx.Self, ctx.Neigh, bufio.NewReader(fd), pipeW, true, nil) + var pkt nncp.Pkt + _, err = xdr.Unmarshal(pipeR, &pkt) + fd.Close() + pipeW.Close() + if err != nil { + ctx.LogE("ack-read-unmarshal", les, err, func(les nncp.LEs) string { + return logMsg(les) + ": unmarshal" + }) + isBad = true + continue + } + if pkt.Type == nncp.PktTypeACK { + ctx.LogI("ack-read-if-ack", les, func(les nncp.LEs) string { + return logMsg(les) + ": it is ACK, skipping" + }) + continue + } + newPktName, err := ctx.TxACK(node, nice, pktName, minSize) + if err != nil { + log.Fatalln(err) + } + acksCreated.WriteString(node.Id.String() + "/" + newPktName + "\n") + } + } + if isBad { + os.Exit(1) + } +} diff -Nru nncp-8.5.0/src/cmd/nncp-bundle/main.go nncp-8.7.2/src/cmd/nncp-bundle/main.go --- nncp-8.5.0/src/cmd/nncp-bundle/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-bundle/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -483,8 +483,10 @@ if err = bufTmp.Flush(); err != nil { log.Fatalln("Error during flushing:", err) } - if err = tmp.Sync(); err != nil { - log.Fatalln("Error during syncing:", err) + if !nncp.NoSync { + if err = tmp.Sync(); err != nil { + log.Fatalln("Error during syncing:", err) + } } if err = tmp.Close(); err != nil { log.Fatalln("Error during closing:", err) diff -Nru nncp-8.5.0/src/cmd/nncp-call/main.go nncp-8.7.2/src/cmd/nncp-call/main.go --- nncp-8.5.0/src/cmd/nncp-call/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-call/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -71,6 +71,7 @@ autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") + autoTossNoACK = flag.Bool("autotoss-noack", false, "Do not process \"ack\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -217,6 +218,7 @@ *autoTossNoExec, *autoTossNoTrns, *autoTossNoArea, + *autoTossNoACK, ) } diff -Nru nncp-8.5.0/src/cmd/nncp-caller/main.go nncp-8.7.2/src/cmd/nncp-caller/main.go --- nncp-8.5.0/src/cmd/nncp-caller/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-caller/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -59,6 +59,7 @@ autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") + autoTossNoACK = flag.Bool("autotoss-noack", false, "Do not process \"ack\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -219,6 +220,7 @@ call.AutoTossNoExec || *autoTossNoExec, call.AutoTossNoTrns || *autoTossNoTrns, call.AutoTossNoArea || *autoTossNoArea, + call.AutoTossNoACK || *autoTossNoACK, ) } diff -Nru nncp-8.5.0/src/cmd/nncp-daemon/main.go nncp-8.7.2/src/cmd/nncp-daemon/main.go --- nncp-8.5.0/src/cmd/nncp-daemon/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-daemon/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -158,6 +158,7 @@ autoTossNoExec = flag.Bool("autotoss-noexec", false, "Do not process \"exec\" packets during tossing") autoTossNoTrns = flag.Bool("autotoss-notrns", false, "Do not process \"trns\" packets during tossing") autoTossNoArea = flag.Bool("autotoss-noarea", false, "Do not process \"area\" packets during tossing") + autoTossNoACK = flag.Bool("autotoss-noack", false, "Do not process \"ack\" packets during tossing") ) log.SetFlags(log.Lshortfile) flag.Usage = usage @@ -217,6 +218,7 @@ *autoTossNoExec, *autoTossNoTrns, *autoTossNoArea, + *autoTossNoACK, ) } <-nodeIdC // call completion @@ -303,6 +305,7 @@ *autoTossNoExec, *autoTossNoTrns, *autoTossNoArea, + *autoTossNoACK, ) } <-nodeIdC // call completion diff -Nru nncp-8.5.0/src/cmd/nncp-pkt/main.go nncp-8.7.2/src/cmd/nncp-pkt/main.go --- nncp-8.5.0/src/cmd/nncp-pkt/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-pkt/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -74,6 +74,8 @@ payloadType = "exec uncompressed" case nncp.PktTypeArea: payloadType = "area" + case nncp.PktTypeACK: + payloadType = "acknowledgement" } var path string switch pkt.Type { @@ -92,6 +94,8 @@ if areaId, err := nncp.AreaIdFromString(path); err == nil { path = fmt.Sprintf("%s (%s)", path, ctx.AreaName(areaId)) } + case nncp.PktTypeACK: + path = nncp.Base32Codec.EncodeToString(pkt.Path[:pkt.PathLen]) default: path = string(pkt.Path[:pkt.PathLen]) } diff -Nru nncp-8.5.0/src/cmd/nncp-reass/main.go nncp-8.7.2/src/cmd/nncp-reass/main.go --- nncp-8.5.0/src/cmd/nncp-reass/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-reass/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -237,8 +237,10 @@ log.Fatalln("Can not flush:", err) } if tmp != nil { - if err = tmp.Sync(); err != nil { - log.Fatalln("Can not sync:", err) + if !nncp.NoSync { + if err = tmp.Sync(); err != nil { + log.Fatalln("Can not sync:", err) + } } if err = tmp.Close(); err != nil { log.Fatalln("Can not close:", err) diff -Nru nncp-8.5.0/src/cmd/nncp-rm/main.go nncp-8.7.2/src/cmd/nncp-rm/main.go --- nncp-8.5.0/src/cmd/nncp-rm/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-rm/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -21,6 +21,7 @@ import ( "flag" "fmt" + "io" "log" "os" "path/filepath" @@ -35,15 +36,14 @@ func usage() { fmt.Fprintf(os.Stderr, nncp.UsageHeader()) fmt.Fprintf(os.Stderr, "nncp-rm -- remove packet\n\n") - fmt.Fprintf(os.Stderr, "Usage: %s [options] -tmp\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "Usage: %s [options] [-older X] -tmp\n", os.Args[0]) fmt.Fprintf(os.Stderr, " %s [options] -lock\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -part\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -seen\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -nock\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -hdr\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -area\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} {-rx|-tx}\n", os.Args[0]) - fmt.Fprintf(os.Stderr, " %s [options] {-all|-node NODE} -pkt PKT\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [-older X] {-all|-node NODE} -part\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [-older X] {-all|-node NODE} -seen\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [-older X] {-all|-node NODE} -nock\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [-older X] {-all|-node NODE} -area\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [-older X] {-all|-node NODE} {-rx|-tx} [-hdr]\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " %s [options] [-older X] {-all|-node NODE} -pkt < ...\n", os.Args[0]) fmt.Fprintln(os.Stderr, "-older option's time units are: (s)econds, (m)inutes, (h)ours, (d)ays") fmt.Fprintln(os.Stderr, "Options:") flag.PrintDefaults() @@ -52,20 +52,20 @@ func main() { var ( cfgPath = flag.String("cfg", nncp.DefaultCfgPath, "Path to configuration file") - doAll = flag.Bool("all", false, "Apply remove rules to all nodes") - doTmp = flag.Bool("tmp", false, "Remove all temporary files") - doHdr = flag.Bool("hdr", false, "Remove all hdr/ files") + doTmp = flag.Bool("tmp", false, "Remove temporary files") doLock = flag.Bool("lock", false, "Remove all lock files") - nodeRaw = flag.String("node", "", "Node to remove files in") - doRx = flag.Bool("rx", false, "Process received packets") - doTx = flag.Bool("tx", false, "Process transfered packets") + doAll = flag.Bool("all", false, "Process all nodes") + nodeRaw = flag.String("node", "", "Process only that node") + doRx = flag.Bool("rx", false, "Process inbound packets") + doTx = flag.Bool("tx", false, "Process outbound packets") doPart = flag.Bool("part", false, "Remove only .part files") doSeen = flag.Bool("seen", false, "Remove only seen/ files") doNoCK = flag.Bool("nock", false, "Remove only .nock files") + doHdr = flag.Bool("hdr", false, "Remove only hdr/ files") doArea = flag.Bool("area", false, "Remove only area/* seen files") older = flag.String("older", "", "XXX{smhd}: only older than XXX number of time units") dryRun = flag.Bool("dryrun", false, "Do not actually remove files") - pktRaw = flag.String("pkt", "", "Packet to remove") + doPkt = flag.Bool("pkt", false, "Remove only that packets") spoolPath = flag.String("spool", "", "Override path to spool") quiet = flag.Bool("quiet", false, "Print only errors") debug = flag.Bool("debug", false, "Print debug messages") @@ -114,6 +114,21 @@ } oldBoundary := time.Second * time.Duration(oldBoundaryRaw) + pkts := make(map[string]struct{}) + if *doPkt { + raw, err := io.ReadAll(os.Stdin) + if err != nil { + log.Fatalln("can not read -pkt from stdin:", err) + } + for _, line := range strings.Fields(string(raw)) { + if len(line) == 0 { + continue + } + cols := strings.Split(line, "/") + pkts[cols[len(cols)-1]] = struct{}{} + } + } + now := time.Now() if *doTmp { err = filepath.Walk( @@ -144,6 +159,7 @@ } return } + if *doLock { err = filepath.Walk(ctx.Spool, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -168,6 +184,7 @@ } return } + var nodeId *nncp.NodeId if *nodeRaw == "" { if !*doAll { @@ -181,6 +198,7 @@ } nodeId = node.Id } + for _, node := range ctx.Neigh { if nodeId != nil && node.Id != nodeId { continue @@ -190,114 +208,146 @@ if _, err := os.Stat(p); err != nil && os.IsNotExist(err) { return nil } - return filepath.Walk(p, - func(path string, info os.FileInfo, err error) error { - if err != nil { - return err + dir, err := os.Open(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer dir.Close() + for { + entries, err := dir.ReadDir(1 << 10) + if err != nil { + if err == io.EOF { + break } - if info.IsDir() { - return nil + return err + } + for _, entry := range entries { + if entry.IsDir() { + continue } + pth := filepath.Join(p, entry.Name()) logMsg := func(les nncp.LEs) string { - return fmt.Sprintf("File %s: removed", path) + return fmt.Sprintf("File %s: removed", pth) + } + if len(pkts) > 0 { + if _, exists := pkts[filepath.Base(entry.Name())]; exists { + ctx.LogI("rm", nncp.LEs{{K: "File", V: pth}}, logMsg) + if !*dryRun { + os.Remove(nncp.JobPath2Hdr(pth)) + if err = os.Remove(pth); err != nil { + return err + } + } + } + continue + } + info, err := entry.Info() + if err != nil { + return err } if now.Sub(info.ModTime()) < oldBoundary { - ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { - return fmt.Sprintf("File %s: too fresh, skipping", path) + ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: pth}}, func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", pth) }) - return nil + continue } - if (*doNoCK && strings.HasSuffix(info.Name(), nncp.NoCKSuffix)) || - (*doPart && strings.HasSuffix(info.Name(), nncp.PartSuffix)) { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if (*doNoCK && strings.HasSuffix(entry.Name(), nncp.NoCKSuffix)) || + (*doPart && strings.HasSuffix(entry.Name(), nncp.PartSuffix)) { + ctx.LogI("rm", nncp.LEs{{K: "File", V: pth}}, logMsg) if *dryRun { - return nil + continue } - return os.Remove(path) - } - if *pktRaw != "" && filepath.Base(info.Name()) == *pktRaw { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { - return nil + if err = os.Remove(pth); err != nil { + return err } - return os.Remove(path) } if !*doSeen && !*doNoCK && !*doHdr && !*doPart && (*doRx || *doTx) && ((*doRx && xx == nncp.TRx) || (*doTx && xx == nncp.TTx)) { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + ctx.LogI("rm", nncp.LEs{{K: "File", V: pth}}, logMsg) if *dryRun { - return nil + continue + } + os.Remove(nncp.JobPath2Hdr(pth)) + if err = os.Remove(pth); err != nil { + return err } - return os.Remove(path) } - return nil - }) + } + } + return nil } - if *pktRaw != "" || *doRx || *doNoCK || *doPart { + if len(pkts) > 0 || *doRx || *doNoCK || *doPart { if err = remove(nncp.TRx); err != nil { log.Fatalln("Can not remove:", err) } } - if *pktRaw != "" || *doTx || *doHdr { + if len(pkts) > 0 || *doTx { if err = remove(nncp.TTx); err != nil { log.Fatalln("Can not remove:", err) } } - removeSub := func(p string, everything bool) error { - return filepath.Walk( - p, func(path string, info os.FileInfo, err error) error { - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - if info.IsDir() { + removeSub := func(p string) error { + return filepath.Walk(p, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { return nil } - logMsg := func(les nncp.LEs) string { - return fmt.Sprintf("File %s: removed", path) - } - if everything { - ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) - if *dryRun { - return nil - } - return os.Remove(path) + return err + } + if info.IsDir() { + return nil + } + logMsg := func(les nncp.LEs) string { + return fmt.Sprintf("File %s: removed", path) + } + if len(pkts) > 0 { + if _, exists := pkts[filepath.Base(info.Name())]; !exists { + return nil } - if now.Sub(info.ModTime()) < oldBoundary { - ctx.LogD( - "rm-skip", nncp.LEs{{K: "File", V: path}}, - func(les nncp.LEs) string { - return fmt.Sprintf("File %s: too fresh, skipping", path) - }, - ) - } else if !*dryRun { - return os.Remove(path) + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil } + return os.Remove(path) + } + if now.Sub(info.ModTime()) < oldBoundary { + ctx.LogD( + "rm-skip", nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }, + ) return nil - }, - ) + } + ctx.LogI("rm", nncp.LEs{{K: "File", V: path}}, logMsg) + if *dryRun { + return nil + } + return os.Remove(path) + }) } - if *doRx || *doSeen { + if len(pkts) > 0 || *doSeen { if err = removeSub(filepath.Join( ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.SeenDir, - ), *doSeen); err != nil { + )); err != nil { log.Fatalln("Can not remove:", err) } } - if *doRx || *doHdr { + if *doRx && *doHdr { if err = removeSub(filepath.Join( ctx.Spool, node.Id.String(), string(nncp.TRx), nncp.HdrDir, - ), *doHdr); err != nil { + )); err != nil { log.Fatalln("Can not remove:", err) } } - if *doTx || *doHdr { + if *doTx && *doHdr { if err = removeSub(filepath.Join( ctx.Spool, node.Id.String(), string(nncp.TTx), nncp.HdrDir, - ), *doHdr); err != nil { + )); err != nil { log.Fatalln("Can not remove:", err) } } @@ -315,9 +365,12 @@ return nil } if now.Sub(info.ModTime()) < oldBoundary { - ctx.LogD("rm-skip", nncp.LEs{{K: "File", V: path}}, func(les nncp.LEs) string { - return fmt.Sprintf("File %s: too fresh, skipping", path) - }) + ctx.LogD( + "rm-skip", nncp.LEs{{K: "File", V: path}}, + func(les nncp.LEs) string { + return fmt.Sprintf("File %s: too fresh, skipping", path) + }, + ) return nil } ctx.LogI( diff -Nru nncp-8.5.0/src/cmd/nncp-toss/main.go nncp-8.7.2/src/cmd/nncp-toss/main.go --- nncp-8.5.0/src/cmd/nncp-toss/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-toss/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -49,6 +49,7 @@ noExec = flag.Bool("noexec", false, "Do not process \"exec\" packets") noTrns = flag.Bool("notrns", false, "Do not process \"transitional\" packets") noArea = flag.Bool("noarea", false, "Do not process \"area\" packets") + noACK = flag.Bool("noack", false, "Do not process \"ack\" packets") spoolPath = flag.String("spool", "", "Override path to spool") logPath = flag.String("log", "", "Override path to logfile") quiet = flag.Bool("quiet", false, "Print only errors") @@ -110,14 +111,14 @@ node.Id, nncp.TRx, nice, - *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, + *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, *noACK, ) || isBad if nodeId == *ctx.SelfId { isBad = ctx.Toss( node.Id, nncp.TTx, nice, - *dryRun, false, true, true, true, true, *noArea, + *dryRun, false, true, true, true, true, *noArea, *noACK, ) || isBad } } @@ -150,14 +151,14 @@ nodeId, nncp.TRx, nice, - *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, + *dryRun, *doSeen, *noFile, *noFreq, *noExec, *noTrns, *noArea, *noACK, ) if *nodeId == *ctx.SelfId { ctx.Toss( nodeId, nncp.TTx, nice, - *dryRun, false, true, true, true, true, *noArea, + *dryRun, false, true, true, true, true, *noArea, *noACK, ) } } diff -Nru nncp-8.5.0/src/cmd/nncp-trns/main.go nncp-8.7.2/src/cmd/nncp-trns/main.go --- nncp-8.5.0/src/cmd/nncp-trns/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-trns/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -137,7 +137,7 @@ if err != nil { panic(err) } - if _, _, err = ctx.Tx( + if _, _, _, err = ctx.Tx( node, pktTrns, nice, diff -Nru nncp-8.5.0/src/cmd/nncp-xfer/main.go nncp-8.7.2/src/cmd/nncp-xfer/main.go --- nncp-8.5.0/src/cmd/nncp-xfer/main.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/cmd/nncp-xfer/main.go 2022-03-13 14:33:13.000000000 +0000 @@ -209,6 +209,24 @@ ctx.NodeName(nodeId), filename, ) } + if _, err = os.Stat(filepath.Join( + ctx.Spool, + nodeId.String(), + string(nncp.TRx), + nncp.SeenDir, + fiInt.Name(), + )); err == nil || !os.IsNotExist(err) { + ctx.LogI("xfer-rx-seen", les, func(les nncp.LEs) string { + return logMsg(les) + ": packet already seen" + }) + if !*keep { + if err = os.Remove(filename); err != nil { + ctx.LogE("xfer-rx-remove", les, err, logMsg) + isBad = true + } + } + continue + } fd, err := os.Open(filename) if err != nil { ctx.LogE("xfer-rx-open", les, err, func(les nncp.LEs) string { @@ -266,7 +284,7 @@ fd.Close() continue } - if _, err = fd.Seek(0, 0); err != nil { + if _, err = fd.Seek(0, io.SeekStart); err != nil { log.Fatalln(err) } tmp, err := ctx.NewTmpFileWHash() @@ -284,7 +302,7 @@ w.CloseWithError(err) } }() - if _, err = nncp.CopyProgressed( + _, err = nncp.CopyProgressed( tmp.W, r, "Rx", append( les, @@ -292,13 +310,24 @@ nncp.LE{K: "FullSize", V: fiInt.Size()}, ), ctx.ShowPrgrs, - ); err != nil { + ) + fd.Close() + if err != nil { ctx.LogE("xfer-rx", les, err, logMsg) + tmp.Cancel() isBad = true + continue } - fd.Close() - if isBad { + if err = tmp.W.Flush(); err != nil { + ctx.LogE("xfer-rx", les, err, logMsg) tmp.Cancel() + isBad = true + continue + } + if tmp.Checksum() != fiInt.Name() { + ctx.LogE("xfer-rx", les, errors.New("checksum mismatch"), logMsg) + tmp.Cancel() + isBad = true continue } if err = tmp.Commit(filepath.Join( @@ -425,14 +454,6 @@ }) continue } - if _, err = os.Stat(filepath.Join( - dstPath, nncp.SeenDir, pktName, - )); err == nil || !os.IsNotExist(err) { - ctx.LogD("xfer-tx-seen", les, func(les nncp.LEs) string { - return logMsg(les) + ": already seen" - }) - continue - } tmp, err := nncp.TempFile(dstPath, "xfer") if err != nil { ctx.LogE("xfer-tx-mktemp", les, err, func(les nncp.LEs) string { @@ -477,13 +498,15 @@ isBad = true continue } - if err = tmp.Sync(); err != nil { - tmp.Close() - ctx.LogE("xfer-tx-sync", les, err, func(les nncp.LEs) string { - return logMsg(les) + ": syncing" - }) - isBad = true - continue + if !nncp.NoSync { + if err = tmp.Sync(); err != nil { + tmp.Close() + ctx.LogE("xfer-tx-sync", les, err, func(les nncp.LEs) string { + return logMsg(les) + ": syncing" + }) + isBad = true + continue + } } if err = tmp.Close(); err != nil { ctx.LogE("xfer-tx-close", les, err, func(les nncp.LEs) string { diff -Nru nncp-8.5.0/src/go.mod nncp-8.7.2/src/go.mod --- nncp-8.5.0/src/go.mod 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/go.mod 2022-03-13 14:33:13.000000000 +0000 @@ -10,17 +10,18 @@ github.com/google/btree v1.0.1 // indirect github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/hjson/hjson-go v3.1.0+incompatible - github.com/klauspost/compress v1.13.6 + github.com/klauspost/compress v1.14.4 + github.com/klauspost/cpuid/v2 v2.0.11 // indirect github.com/yggdrasil-network/yggdrasil-go v0.4.2 go.cypherpunks.ru/balloon v1.1.1 go.cypherpunks.ru/recfile v0.4.3 - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - golang.org/x/net v0.0.0-20210614182718-04defd469f4e - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e + golang.org/x/crypto v0.0.0-20220214200702-86341886e292 + golang.org/x/net v0.0.0-20220225172249-27dd8689420f + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect inet.af/netstack v0.0.0-20211120045802-8aa80cf23d3c - lukechampine.com/blake3 v1.1.6 + lukechampine.com/blake3 v1.1.7 ) go 1.13 diff -Nru nncp-8.5.0/src/go.sum nncp-8.7.2/src/go.sum --- nncp-8.5.0/src/go.sum 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/go.sum 2022-03-13 14:33:13.000000000 +0000 @@ -27,10 +27,11 @@ github.com/hjson/hjson-go v3.1.0+incompatible h1:DY/9yE8ey8Zv22bY+mHV1uk2yRy0h8tKhZ77hEdi0Aw= github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/kardianos/minwinsvc v1.0.0/go.mod h1:Bgd0oc+D0Qo3bBytmNtyRKVlp85dAloLKhfxanPFFRc= -github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/compress v1.14.4 h1:eijASRJcobkVtSt81Olfh7JX43osYLwy5krOJo6YEu4= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.11 h1:i2lw1Pm7Yi/4O6XCSyJWqEHI2MDw2FzUK6o/D21xn2A= +github.com/klauspost/cpuid/v2 v2.0.11/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -61,14 +62,15 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210506145944-38f3c27a63bf/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292 h1:f+lwQ+GtmgoY+A2YaQxlSOnDjXcQ7ZRLWOHbC6HtRqE= +golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f h1:oA4XRj0qtSt8Yo1Zms0CUlsT3KG69V2UGQWPBxujDmc= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -85,8 +87,9 @@ golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY= @@ -95,6 +98,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7-0.20210503195748-5c7c50ebbd4f/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -105,5 +109,5 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= inet.af/netstack v0.0.0-20211120045802-8aa80cf23d3c h1:nr31qYr+91rWD8klUkPx3eGTZzumCC414UJG1QRKZTc= inet.af/netstack v0.0.0-20211120045802-8aa80cf23d3c/go.mod h1:KOJdAzQzMLKzwFEdOOnrnSrLIhaFVB+NQoME/e5wllA= -lukechampine.com/blake3 v1.1.6 h1:H3cROdztr7RCfoaTpGZFQsrqvweFLrqS73j7L7cmR5c= -lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= +lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff -Nru nncp-8.5.0/src/nncp.go nncp-8.7.2/src/nncp.go --- nncp-8.5.0/src/nncp.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/nncp.go 2022-03-13 14:33:13.000000000 +0000 @@ -40,7 +40,7 @@ const Base32Encoded32Len = 52 var ( - Version string = "8.5.0" + Version string = "8.7.2" Base32Codec *base32.Encoding = base32.StdEncoding.WithPadding(base32.NoPadding) ) diff -Nru nncp-8.5.0/src/pkt.go nncp-8.7.2/src/pkt.go --- nncp-8.5.0/src/pkt.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/pkt.go 2022-03-13 14:33:13.000000000 +0000 @@ -44,6 +44,7 @@ PktTypeTrns PktType = iota PktTypeExecFat PktType = iota PktTypeArea PktType = iota + PktTypeACK PktType = iota MaxPathSize = 1<<8 - 1 diff -Nru nncp-8.5.0/src/sp.go nncp-8.7.2/src/sp.go --- nncp-8.5.0/src/sp.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/sp.go 2022-03-13 14:33:13.000000000 +0000 @@ -1114,7 +1114,7 @@ return nil } -func (state *SPState) Wait() { +func (state *SPState) Wait() bool { state.wg.Wait() close(state.payloads) close(state.pings) @@ -1130,12 +1130,15 @@ if txDuration > 0 { state.TxSpeed = state.TxBytes / txDuration } + nothingLeft := len(state.queueTheir) == 0 for _, s := range state.fds { + nothingLeft = false s.fd.Close() } for pktName := range state.progressBars { ProgressKill(pktName) } + return nothingLeft } func (state *SPState) ProcessSP(payload []byte) ([][]byte, error) { @@ -1461,13 +1464,15 @@ humanize.IBytes(uint64(fullsize)), ) } - err = fd.Sync() - if err != nil { - state.Ctx.LogE("sp-file-sync", lesp, err, func(les LEs) string { - return logMsg(les) + ": syncing" - }) - state.closeFd(filePathPart) - continue + if !NoSync { + err = fd.Sync() + if err != nil { + state.Ctx.LogE("sp-file-sync", lesp, err, func(les LEs) string { + return logMsg(les) + ": syncing" + }) + state.closeFd(filePathPart) + continue + } } if hasherAndOffset != nil { delete(state.fileHashers, filePath) diff -Nru nncp-8.5.0/src/tmp.go nncp-8.7.2/src/tmp.go --- nncp-8.5.0/src/tmp.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/tmp.go 2022-03-13 14:33:13.000000000 +0000 @@ -28,6 +28,12 @@ "time" ) +var NoSync bool + +func init() { + NoSync = os.Getenv(CfgNoSync) != "" +} + func TempFile(dir, prefix string) (*os.File, error) { // Assume that probability of suffix collision is negligible suffix := strconv.FormatInt(time.Now().UnixNano()+int64(os.Getpid()), 16) @@ -77,6 +83,9 @@ } func DirSync(dirPath string) error { + if NoSync { + return nil + } fd, err := os.Open(dirPath) if err != nil { return err @@ -102,9 +111,11 @@ tmp.Fd.Close() return err } - if err = tmp.Fd.Sync(); err != nil { - tmp.Fd.Close() - return err + if !NoSync { + if err = tmp.Fd.Sync(); err != nil { + tmp.Fd.Close() + return err + } } if err = tmp.Fd.Close(); err != nil { return err diff -Nru nncp-8.5.0/src/toss.go nncp-8.7.2/src/toss.go --- nncp-8.5.0/src/toss.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/toss.go 2022-03-13 14:33:13.000000000 +0000 @@ -91,7 +91,7 @@ pktSize uint64, jobPath string, decompressor *zstd.Decoder, - dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, noACK bool, ) error { defer pipeR.Close() sendmail := ctx.Neigh[*ctx.SelfId].Exec["sendmail"] @@ -329,16 +329,18 @@ }) return err } - if err = tmp.Sync(); err != nil { - tmp.Close() - ctx.LogE("rx-sync", les, err, func(les LEs) string { - return fmt.Sprintf( - "Tossing file %s/%s (%s): %s: syncing", - sender.Name, pktName, - humanize.IBytes(pktSize), dst, - ) - }) - return err + if !NoSync { + if err = tmp.Sync(); err != nil { + tmp.Close() + ctx.LogE("rx-sync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: syncing", + sender.Name, pktName, + humanize.IBytes(pktSize), dst, + ) + }) + return err + } } if err = tmp.Close(); err != nil { ctx.LogE("rx-close", les, err, func(les LEs) string { @@ -619,7 +621,7 @@ if err != nil { panic(err) } - if _, _, err = ctx.Tx( + if _, _, _, err = ctx.Tx( node, pktTrns, nice, @@ -750,7 +752,7 @@ } if nodeId != sender.Id && nodeId != pktEnc.Sender { ctx.LogI("rx-area-echo", lesEcho, logMsgNode) - if _, _, err = ctx.Tx( + if _, _, _, err = ctx.Tx( node, &pkt, nice, @@ -856,7 +858,7 @@ uint64(pktSizeWithoutEnc(int64(pktSize))), "", decompressor, - dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, noACK, ) }() _, _, _, err = PktEncRead( @@ -906,6 +908,66 @@ } } + case PktTypeACK: + if noACK { + return nil + } + hsh := Base32Codec.EncodeToString(pkt.Path[:MTHSize]) + les := append(les, LE{"Type", "ack"}, LE{"Pkt", hsh}) + logMsg := func(les LEs) string { + return fmt.Sprintf("Tossing ack %s/%s: %s", sender.Name, pktName, hsh) + } + ctx.LogD("rx-ack", les, logMsg) + pktPath := filepath.Join(ctx.Spool, sender.Id.String(), string(TTx), hsh) + if _, err := os.Stat(pktPath); err == nil { + if !dryRun { + if err = os.Remove(pktPath); err != nil { + ctx.LogE("rx-ack", les, err, func(les LEs) string { + return logMsg(les) + ": removing packet" + }) + return err + } else if ctx.HdrUsage { + os.Remove(JobPath2Hdr(pktPath)) + } + } + } else { + ctx.LogD("rx-ack", les, func(les LEs) string { + return logMsg(les) + ": already disappeared" + }) + } + if !dryRun && doSeen { + if err := ensureDir(filepath.Dir(jobPath), SeenDir); err != nil { + return err + } + if fd, err := os.Create(jobPath2Seen(jobPath)); err == nil { + fd.Close() + if err = DirSync(filepath.Dir(jobPath)); err != nil { + ctx.LogE("rx-dirsync", les, err, func(les LEs) string { + return fmt.Sprintf( + "Tossing file %s/%s (%s): %s: dirsyncing", + sender.Name, pktName, + humanize.IBytes(pktSize), + filepath.Base(jobPath), + ) + }) + return err + } + } + } + if !dryRun { + if err = os.Remove(jobPath); err != nil { + ctx.LogE("rx", les, err, func(les LEs) string { + return logMsg(les) + ": removing job" + }) + return err + } else if ctx.HdrUsage { + os.Remove(JobPath2Hdr(jobPath)) + } + } + ctx.LogI("rx", les, func(les LEs) string { + return fmt.Sprintf("Got ACK packet from %s of %s", sender.Name, hsh) + }) + default: err = errors.New("unknown type") ctx.LogE( @@ -926,7 +988,7 @@ nodeId *NodeId, xx TRxTx, nice uint8, - dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea bool, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, noACK bool, ) bool { dirLock, err := ctx.LockDir(nodeId, "toss") if err != nil { @@ -994,7 +1056,7 @@ uint64(pktSizeWithoutEnc(job.Size)), job.Path, decompressor, - dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, + dryRun, doSeen, noFile, noFreq, noExec, noTrns, noArea, noACK, ) }() pipeWB := bufio.NewWriter(pipeW) @@ -1044,7 +1106,7 @@ func (ctx *Ctx) AutoToss( nodeId *NodeId, nice uint8, - doSeen, noFile, noFreq, noExec, noTrns, noArea bool, + doSeen, noFile, noFreq, noExec, noTrns, noArea, noACK bool, ) (chan struct{}, chan bool) { dw, err := ctx.NewDirWatcher( filepath.Join(ctx.Spool, nodeId.String(), string(TRx)), @@ -1066,7 +1128,7 @@ case <-dw.C: bad = !ctx.Toss( nodeId, TRx, nice, false, - doSeen, noFile, noFreq, noExec, noTrns, noArea) || bad + doSeen, noFile, noFreq, noExec, noTrns, noArea, noACK) || bad } } }() diff -Nru nncp-8.5.0/src/toss_test.go nncp-8.7.2/src/toss_test.go --- nncp-8.5.0/src/toss_test.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/toss_test.go 2022-03-13 14:33:13.000000000 +0000 @@ -113,14 +113,14 @@ continue } ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec-1, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].Exec = make(map[string][]string) ctx.Neigh[*nodeOur.Id].Exec[handle] = []string{"/bin/sh", "-c", "false"} ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } @@ -133,7 +133,7 @@ ), } ctx.Toss(ctx.Self.Id, TRx, DefaultNiceExec, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } @@ -220,13 +220,13 @@ rxPath := filepath.Join(spool, ctx.Self.Id.String(), string(TRx)) os.Rename(filepath.Join(spool, ctx.Self.Id.String(), string(TTx)), rxPath) ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].Incoming = &incomingPath if ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, - false, false, false, false, false, false, false) { + false, false, false, false, false, false, false, false) { return false } if len(dirFiles(rxPath)) != 0 { @@ -303,7 +303,7 @@ os.Rename(filepath.Join(spool, ctx.Self.Id.String(), string(TTx)), rxPath) ctx.Neigh[*nodeOur.Id].Incoming = &incomingPath ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFile, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) expected := make(map[string]struct{}) expected["samefile"] = struct{}{} for i := 0; i < files-1; i++ { @@ -379,13 +379,13 @@ os.Rename(txPath, rxPath) os.MkdirAll(txPath, os.FileMode(0700)) ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(txPath)) != 0 || len(dirFiles(rxPath)) == 0 { return false } ctx.Neigh[*nodeOur.Id].FreqPath = &spool ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(txPath)) != 0 || len(dirFiles(rxPath)) == 0 { return false } @@ -399,7 +399,7 @@ } } ctx.Toss(ctx.Self.Id, TRx, DefaultNiceFreq, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(txPath)) == 0 || len(dirFiles(rxPath)) != 0 { return false } @@ -504,7 +504,7 @@ } } ctx.Toss(ctx.Self.Id, TRx, 123, - false, false, false, false, false, false, false) + false, false, false, false, false, false, false, false) if len(dirFiles(rxPath)) != 0 { return false } diff -Nru nncp-8.5.0/src/tx.go nncp-8.7.2/src/tx.go --- nncp-8.5.0/src/tx.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/tx.go 2022-03-13 14:33:13.000000000 +0000 @@ -57,12 +57,12 @@ src io.Reader, pktName string, areaId *AreaId, -) (*Node, int64, error) { +) (*Node, int64, string, error) { var area *Area if areaId != nil { area = ctx.AreaId2Area[*areaId] if area.Prv == nil { - return nil, 0, errors.New("area has no encryption keys") + return nil, 0, "", errors.New("area has no encryption keys") } } hops := make([]*Node, 0, 1+len(node.Via)) @@ -82,15 +82,15 @@ expectedSize += sizePadCalc(expectedSize, minSize, wrappers) expectedSize = PktEncOverhead + sizeWithTags(expectedSize) if maxSize != 0 && expectedSize > maxSize { - return nil, 0, TooBig + return nil, 0, "", TooBig } if !ctx.IsEnoughSpace(expectedSize) { - return nil, 0, errors.New("is not enough space") + return nil, 0, "", errors.New("is not enough space") } } tmp, err := ctx.NewTmpFileWHash() if err != nil { - return nil, 0, err + return nil, 0, "", err } results := make(chan PktEncWriteResult) @@ -211,7 +211,7 @@ r := <-results if r.err != nil { tmp.Fd.Close() - return nil, 0, r.err + return nil, 0, "", r.err } if r.pktEncRaw != nil { pktEncRaw = r.pktEncRaw @@ -224,7 +224,7 @@ err = tmp.Commit(filepath.Join(nodePath, string(TTx))) os.Symlink(nodePath, filepath.Join(ctx.Spool, lastNode.Name)) if err != nil { - return lastNode, 0, err + return lastNode, 0, "", err } if ctx.HdrUsage { ctx.HdrWrite(pktEncRaw, filepath.Join(nodePath, string(TTx), tmp.Checksum())) @@ -255,18 +255,18 @@ } if err = ensureDir(seenDir); err != nil { ctx.LogE("tx-mkdir", les, err, logMsg) - return lastNode, 0, err + return lastNode, 0, "", err } if fd, err := os.Create(seenPath); err == nil { fd.Close() if err = DirSync(seenDir); err != nil { ctx.LogE("tx-dirsync", les, err, logMsg) - return lastNode, 0, err + return lastNode, 0, "", err } } ctx.LogI("tx-area", les, logMsg) } - return lastNode, payloadSize, err + return lastNode, payloadSize, tmp.Checksum(), err } type DummyCloser struct{} @@ -438,7 +438,7 @@ if err != nil { return err } - _, finalSize, err := ctx.Tx( + _, finalSize, pktName, err := ctx.Tx( node, pkt, nice, srcSize, minSize, maxSize, bufio.NewReader(reader), dstPath, areaId, @@ -450,10 +450,11 @@ {"Src", srcPath}, {"Dst", dstPath}, {"Size", finalSize}, + {"Pkt", pktName}, } logMsg := func(les LEs) string { return fmt.Sprintf( - "File %s (%s) sent to %s:%s", + "File %s (%s) is sent to %s:%s", srcPath, humanize.IBytes(uint64(finalSize)), ctx.NodeName(node.Id), @@ -480,7 +481,7 @@ return err } hsh := MTHNew(0, 0) - _, size, err := ctx.Tx( + _, size, pktName, err := ctx.Tx( node, pkt, nice, 0, minSize, maxSize, io.TeeReader(lr, hsh), @@ -494,10 +495,11 @@ {"Src", srcPath}, {"Dst", path}, {"Size", size}, + {"Pkt", pktName}, } logMsg := func(les LEs) string { return fmt.Sprintf( - "File %s (%s) sent to %s:%s", + "File %s (%s) is sent to %s:%s", srcPath, humanize.IBytes(uint64(size)), ctx.NodeName(node.Id), @@ -541,7 +543,7 @@ return err } metaPktSize := int64(buf.Len()) - _, _, err = ctx.Tx( + _, _, pktName, err := ctx.Tx( node, pkt, nice, @@ -555,10 +557,11 @@ {"Src", srcPath}, {"Dst", path}, {"Size", metaPktSize}, + {"Pkt", pktName}, } logMsg := func(les LEs) string { return fmt.Sprintf( - "File %s (%s) sent to %s:%s", + "File %s (%s) is sent to %s:%s", srcPath, humanize.IBytes(uint64(metaPktSize)), ctx.NodeName(node.Id), @@ -593,7 +596,9 @@ } src := strings.NewReader(dstPath) size := int64(src.Len()) - _, _, err = ctx.Tx(node, pkt, nice, size, minSize, MaxFileSize, src, srcPath, nil) + _, _, pktName, err := ctx.Tx( + node, pkt, nice, size, minSize, MaxFileSize, src, srcPath, nil, + ) les := LEs{ {"Type", "freq"}, {"Node", node.Id}, @@ -601,10 +606,11 @@ {"ReplyNice", int(replyNice)}, {"Src", srcPath}, {"Dst", dstPath}, + {"Pkt", pktName}, } logMsg := func(les LEs) string { return fmt.Sprintf( - "File request from %s:%s to %s sent", + "File request from %s:%s to %s is sent", ctx.NodeName(node.Id), srcPath, dstPath, ) @@ -657,7 +663,9 @@ }(in) in = pr } - _, size, err := ctx.Tx(node, pkt, nice, 0, minSize, maxSize, in, handle, areaId) + _, size, pktName, err := ctx.Tx( + node, pkt, nice, 0, minSize, maxSize, in, handle, areaId, + ) if !noCompress { e := <-compressErr if err == nil { @@ -672,10 +680,11 @@ {"ReplyNice", int(replyNice)}, {"Dst", dst}, {"Size", size}, + {"Pkt", pktName}, } logMsg := func(les LEs) string { return fmt.Sprintf( - "Exec sent to %s@%s (%s)", + "Exec is sent to %s@%s (%s)", ctx.NodeName(node.Id), dst, humanize.IBytes(uint64(size)), ) } @@ -729,3 +738,42 @@ os.Symlink(nodePath, filepath.Join(ctx.Spool, node.Name)) return err } + +func (ctx *Ctx) TxACK( + node *Node, + nice uint8, + hsh string, + minSize int64, +) (pktName string, err error) { + hshRaw, err := Base32Codec.DecodeString(hsh) + if err != nil { + return "", err + } + if len(hshRaw) != MTHSize { + return "", errors.New("Invalid packet id size") + } + pkt, err := NewPkt(PktTypeACK, nice, []byte(hshRaw)) + if err != nil { + return "", err + } + src := bytes.NewReader([]byte{}) + _, _, pktName, err = ctx.Tx( + node, pkt, nice, 0, minSize, MaxFileSize, src, hsh, nil, + ) + les := LEs{ + {"Type", "ack"}, + {"Node", node.Id}, + {"Nice", int(nice)}, + {"Pkt", hsh}, + {"NewPkt", pktName}, + } + logMsg := func(les LEs) string { + return fmt.Sprintf("ACK to %s of %s is sent", ctx.NodeName(node.Id), hsh) + } + if err == nil { + ctx.LogI("tx", les, logMsg) + } else { + ctx.LogE("tx", les, err, logMsg) + } + return +} diff -Nru nncp-8.5.0/src/tx_test.go nncp-8.7.2/src/tx_test.go --- nncp-8.5.0/src/tx_test.go 2022-01-26 10:30:45.000000000 +0000 +++ nncp-8.7.2/src/tx_test.go 2022-03-13 14:33:13.000000000 +0000 @@ -87,7 +87,7 @@ } pkt, err := NewPkt(PktTypeExec, replyNice, []byte(pathSrc)) src := bytes.NewReader(data) - dstNode, _, err := ctx.Tx( + dstNode, _, _, err := ctx.Tx( nodeTgt, pkt, 123, diff -Nru nncp-8.5.0/t/nncp-rm.t nncp-8.7.2/t/nncp-rm.t --- nncp-8.5.0/t/nncp-rm.t 1970-01-01 00:00:00.000000000 +0000 +++ nncp-8.7.2/t/nncp-rm.t 2022-03-13 14:33:13.000000000 +0000 @@ -0,0 +1,278 @@ +#!/bin/sh + +testname=`basename "$0"` +test_description="nncp-rm's behaviour" +. $SHARNESS_TEST_SRCDIR/sharness.sh + +rand32() { + perl <&2 + return 1 + fi + done + local pkts="" + for pkt in `pkts_remove $@` ; do + if ! [ -e $pkt ] ; then + echo unexpectedly removed: $pkt >&2 + return 1 + fi + pkts="$pkts $pkt" + done + PKTS="$pkts" +} + +now=`date +%s` +nncp-cfgnew > cfg +nncp-cfgdir -cfg cfg -dump cfgdir +echo "$PWD/spool" > cfgdir/spool +echo "$PWD/log" > cfgdir/log +neigh=`rand32` +mkdir -p cfgdir/neigh/neigh +for w in id exchpub signpub ; do echo $neigh > cfgdir/neigh/neigh/$w ; done +mkdir -p spool/tmp spool/$neigh/rx/hdr spool/$neigh/rx/seen spool/$neigh/tx/hdr +date_old=$(date -j -f %s +%FT%T $(( $now - (3600 * 24 * 7) ))) + +pkts_old_rx="" +pkts_old_rx_hdr="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/rx/$pkt spool/$neigh/rx/hdr/$pkt + pkts_old_rx="$pkts_old_rx spool/$neigh/rx/$pkt" + pkts_old_rx_hdr="$pkts_old_rx_hdr spool/$neigh/rx/hdr/$pkt" +done + +pkts_old_tx="" +pkts_old_tx_hdr="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/tx/$pkt spool/$neigh/tx/hdr/$pkt + pkts_old_tx="$pkts_old_tx spool/$neigh/tx/$pkt" + pkts_old_tx_hdr="$pkts_old_tx_hdr spool/$neigh/tx/hdr/$pkt" +done + +pkts_old_part="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/rx/$pkt.part + pkts_old_part="$pkts_old_part spool/$neigh/rx/$pkt.part" +done + +pkts_old_nock="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/rx/$pkt.nock + pkts_old_nock="$pkts_old_nock spool/$neigh/rx/$pkt.nock" +done + +pkts_old_seen="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/rx/seen/$pkt + pkts_old_seen="$pkts_old_seen spool/$neigh/rx/seen/$pkt" +done + +pkts_new_rx="" +pkts_new_rx_hdr="" +for pkt in `randpkts` ; do + touch spool/$neigh/rx/$pkt spool/$neigh/rx/hdr/$pkt + pkts_new_rx="$pkts_new_rx spool/$neigh/rx/$pkt" + pkts_new_rx_hdr="$pkts_new_rx_hdr spool/$neigh/rx/hdr/$pkt" +done + +pkts_new_tx="" +pkts_new_tx_hdr="" +for pkt in `randpkts` ; do + touch spool/$neigh/tx/$pkt spool/$neigh/tx/hdr/$pkt + pkts_new_tx="$pkts_new_tx spool/$neigh/tx/$pkt" + pkts_new_tx_hdr="$pkts_new_tx_hdr spool/$neigh/tx/hdr/$pkt" +done + +pkts_new_part="" +for pkt in `randpkts` ; do + touch spool/$neigh/rx/$pkt.part + pkts_new_part="$pkts_new_part spool/$neigh/rx/$pkt.part" +done + +pkts_new_nock="" +for pkt in `randpkts` ; do + touch spool/$neigh/rx/$pkt.nock + pkts_new_nock="$pkts_new_nock spool/$neigh/rx/$pkt.nock" +done + +pkts_new_seen="" +for pkt in `randpkts` ; do + touch spool/$neigh/rx/seen/$pkt + pkts_new_seen="$pkts_new_seen spool/$neigh/rx/seen/$pkt" +done + +pkts_tmp_old="" +for pkt in `randpkts` ; do + touch -d $date_old spool/tmp/$pkt + pkts_tmp_old="$pkts_tmp_old spool/tmp/$pkt" +done + +pkts_tmp_new="" +for pkt in `randpkts` ; do + touch spool/tmp/$pkt + pkts_tmp_new="$pkts_tmp_new spool/tmp/$pkt" +done + +pkts_hdr_excess_old_rx="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/rx/hdr/$pkt + pkts_hdr_excess_old_rx="$pkts_hdr_excess_old_rx spool/$neigh/rx/hdr/$pkt" +done + +pkts_hdr_excess_old_tx="" +for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/tx/hdr/$pkt + pkts_hdr_excess_old_tx="$pkts_hdr_excess_old_tx spool/$neigh/tx/hdr/$pkt" +done + +pkts_hdr_excess_new_rx="" +for pkt in `randpkts` ; do + touch spool/$neigh/rx/hdr/$pkt + pkts_hdr_excess_new_rx="$pkts_hdr_excess_new_rx spool/$neigh/rx/hdr/$pkt" +done + +pkts_hdr_excess_new_tx="" +for pkt in `randpkts` ; do + touch spool/$neigh/tx/hdr/$pkt + pkts_hdr_excess_new_tx="$pkts_hdr_excess_new_tx spool/$neigh/tx/hdr/$pkt" +done + +pkts_area_old="" +pkts_area_new="" +for area in `randpkts` ; do + mkdir -p spool/$neigh/area/$area + for pkt in `randpkts` ; do + touch -d $date_old spool/$neigh/area/$area/$pkt + pkts_area_old="$pkts_area_old spool/$neigh/area/$area/$pkt" + done + for pkt in `randpkts` ; do + touch spool/$neigh/area/$area/$pkt + pkts_area_new="$pkts_area_new spool/$neigh/area/$area/$pkt" + done +done + +pkts_specified="" +for pkt in `randpkts` ; do + touch spool/$neigh/rx/$pkt + pkts_specified="$pkts_specified spool/$neigh/rx/$pkt" +done +for pkt in `randpkts` ; do + touch spool/$neigh/tx/$pkt + pkts_specified="$pkts_specified spool/$neigh/tx/$pkt" +done +for pkt in `randpkts` ; do + touch spool/$neigh/rx/seen/$pkt + pkts_specified="$pkts_specified spool/$neigh/rx/seen/$pkt" +done + +PKTS=" +$pkts_old_rx +$pkts_old_rx_hdr +$pkts_old_tx +$pkts_old_tx_hdr +$pkts_old_part +$pkts_old_nock +$pkts_old_seen +$pkts_new_rx +$pkts_new_rx_hdr +$pkts_new_tx +$pkts_new_tx_hdr +$pkts_new_part +$pkts_new_nock +$pkts_new_seen +$pkts_tmp_old +$pkts_tmp_new +$pkts_hdr_excess_old_rx +$pkts_hdr_excess_old_tx +$pkts_hdr_excess_new_rx +$pkts_hdr_excess_new_tx +$pkts_area_old +$pkts_area_new +$pkts_specified +" +rmcmd="nncp-rm -quiet -cfg cfgdir -all" +older="-older 6d" + +$rmcmd -pkt <