diff --git a/.gitignore b/.gitignore index 70061765..b8e90ca4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ aclocal.m4 /help-rsync*.h /default-cvsignore.h /default-dont-compress.h +/daemon-parm.h /.md2man-works /autom4te*.cache /confdefs.h @@ -48,5 +49,7 @@ aclocal.m4 /testsuite/devices-fake.test /testsuite/xattrs-hlink.test /patches -/SaVeDiR +/patches.gen +/build +/auto-build-save .deps diff --git a/Makefile.in b/Makefile.in index 6ca7d9eb..672fcc47 100644 --- a/Makefile.in +++ b/Makefile.in @@ -1,5 +1,4 @@ -# Makefile for rsync. This is processed by configure to produce the final -# Makefile +# The Makefile for rsync (configure creates it from Makefile.in). prefix=@prefix@ datarootdir=@datarootdir@ @@ -27,8 +26,6 @@ MKDIR_P=@MKDIR_P@ VPATH=$(srcdir) SHELL=/bin/sh -VERSION=@RSYNC_VERSION@ - .SUFFIXES: .SUFFIXES: .c .o @@ -38,7 +35,7 @@ ASM_x86_64=lib/md5-asm-x86_64.o GENFILES=configure.sh aclocal.m4 config.h.in proto.h proto.h-tstamp rsync.1 rsync.1.html \ rsync-ssl.1 rsync-ssl.1.html rsyncd.conf.5 rsyncd.conf.5.html HEADERS=byteorder.h config.h errcode.h proto.h rsync.h ifuncs.h itypes.h inums.h \ - lib/pool_alloc.h lib/mdigest.h lib/md-defines.h + lib/pool_alloc.h lib/mdigest.h lib/md-defines.h version.h LIBOBJ=lib/wildmatch.o lib/compat.o lib/snprintf.o lib/mdfour.o lib/md5.o \ lib/permstring.o lib/pool_alloc.o lib/sysacls.o lib/sysxattrs.o @LIBOBJS@ zlib_OBJS=zlib/deflate.o zlib/inffast.o zlib/inflate.o zlib/inftrees.o \ @@ -77,7 +74,7 @@ all: Makefile rsync$(EXEEXT) stunnel-rsyncd.conf @MAKE_MAN@ install: all -${MKDIR_P} ${DESTDIR}${bindir} ${INSTALLCMD} ${INSTALL_STRIP} -m 755 rsync$(EXEEXT) ${DESTDIR}${bindir} - ${INSTALLCMD} -m 755 rsync-ssl ${DESTDIR}${bindir} + ${INSTALLCMD} -m 755 $(srcdir)/rsync-ssl ${DESTDIR}${bindir} -${MKDIR_P} ${DESTDIR}${mandir}/man1 -${MKDIR_P} ${DESTDIR}${mandir}/man5 if test -f rsync.1; then ${INSTALLMAN} -m 644 rsync.1 ${DESTDIR}${mandir}/man1; fi @@ -104,7 +101,7 @@ $(CHECK_OBJS): $(HEADERS) tls.o xattrs.o: lib/sysxattrs.h options.o: latest-year.h help-rsync.h help-rsyncd.h exclude.o: default-cvsignore.h -loadparm.o: default-dont-compress.h +loadparm.o: default-dont-compress.h daemon-parm.h flist.o: rounding.h @@ -114,6 +111,9 @@ default-cvsignore.h default-dont-compress.h: rsync.1.md define-from-md.awk help-rsync.h help-rsyncd.h: rsync.1.md help-from-md.awk $(AWK) -f $(srcdir)/help-from-md.awk -v hfile=$@ $(srcdir)/rsync.1.md +daemon-parm.h: daemon-parm.txt daemon-parm.awk + $(AWK) -f $(srcdir)/daemon-parm.awk $(srcdir)/daemon-parm.txt + rounding.h: rounding.c rsync.h proto.h @for r in 0 1 3; do \ if $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o rounding -DEXTRA_ROUNDING=$$r -I. $(srcdir)/rounding.c >rounding.out 2>&1; then \ @@ -203,6 +203,10 @@ reconfigure: configure.sh ./config.status --recheck ./config.status +.PHONY: restatus +restatus: + ./config.status + Makefile: Makefile.in config.status configure.sh config.h.in @if test -f Makefile; then cp -p Makefile Makefile.old; else touch Makefile.old; fi @./config.status @@ -227,19 +231,19 @@ proto: proto.h-tstamp proto.h: proto.h-tstamp @if test -f proto.h; then :; else cp -p $(srcdir)/proto.h .; fi -proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h - $(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c +proto.h-tstamp: $(srcdir)/*.c $(srcdir)/lib/compat.c config.h daemon-parm.h + $(AWK) -f $(srcdir)/mkproto.awk $(srcdir)/*.c $(srcdir)/lib/compat.c daemon-parm.h .PHONY: man man: rsync.1 rsync-ssl.1 rsyncd.conf.5 -rsync.1: rsync.1.md md2man NEWS.md Makefile +rsync.1: rsync.1.md md2man version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsync.1.md -rsync-ssl.1: rsync-ssl.1.md md2man NEWS.md Makefile +rsync-ssl.1: rsync-ssl.1.md md2man version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsync-ssl.1.md -rsyncd.conf.5: rsyncd.conf.5.md md2man NEWS.md Makefile +rsyncd.conf.5: rsyncd.conf.5.md md2man version.h Makefile @$(srcdir)/maybe-make-man $(srcdir) rsyncd.conf.5.md .PHONY: clean diff --git a/NEWS.md b/NEWS.md index bfbaff22..8a104282 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,10 +1,84 @@ + + +# NEWS for rsync 3.2.2 (4 Jul 2020) + +## Changes in this version: + +### BUG FIXES: + + - Avoid a crash when a daemon module enables `transfer logging` without + setting a `log format` value. + + - Fixed installing rsync-ssl script from an alternate build dir. + + - Fixed the updating of configure.sh from an alternate build dir. + + - Apple requires the asm function name to begin with an underscore. + + - Avoid a test failure in the daemon test when --atimes is disabled. + +### ENHANCEMENTS: + + - Allow the server side to restrict checksum & compression choices via the + same environment variables the client uses. The env vars can be divided + into "client list & server list" by the "`&`" char or the same list can + apply to both. + + - Simplify how the negotiation environment variables apply when interacting + with an older rsync and also when a list contains only invalid names. + + - Do not allow a negotiated checksum or compression choice of "none" unless + the user authorized it via an environment variable or command-line option. + + - Added the `--max-alloc=SIZE` option to be able to override the memory + allocator's sanity-check limit. It defaults to 1G (as before) but the error + message when exceeding it specifically mentions the new option so that you + can differentiate an out-of-memory error from a failure of this limit. It + also allows you to specify the value via the RSYNC_MAX_ALLOC environment + variable. + + - Add the "open atime" daemon parameter to allow a daemon to always enable or + disable the use of O_NOATIME (the default is to let the user control it). + + - The default systemd config was changed to remove the `ProtectHome=on` + setting since rsync is often used to serve files in /home and /root and this + seemed a bit too strict. Feel free to use `systemctl edit rsync` to add + that restriction (or maybe `ProtectHome=read-only`), if you like. See the + 3.2.0 NEWS for the other restrictions that were added compared to 3.1.3. + + - The memory allocation functions now automatically check for a failure and + die when out of memory. This eliminated some caller-side check-and-die + code and added some missing sanity-checking of allocations. + + - Put optimizations into their own list in the `--version` output. + + - Improved the man page a bit more. + +### PACKAGING RELATED: + + - Prepared the checksum code for an upcoming xxHash release that provides new + XXH3 (64-bit) & XXH128 (128-bit) checksum routines. These will not be + compiled into rsync until the xxhash v0.8.0 include files are installed on + the build host, and that release is a few weeks away at the time this was + written. So, if it's now the future and you have packaged and installed + xxhash-0.8.0-devel, a fresh rebuild of rsync 3.2.2 will give you the new + checksum routines. Just make sure that the new rsync package depends on + xxhash >= 0.8.0. + +### DEVELOPER RELATED: + + - Moved the version number out of configure.ac into its own version.h file so + that we don't need to reconfigure just because the version number changes. + + - Moved the daemon parameter list into daemon-parm.txt so that an awk script + can create the interrelated structs and accessors that loadparm.c needs. + +------------------------------------------------------------------------------ # NEWS for rsync 3.2.1 (22 Jun 2020) -Protocol: 31 (unchanged) - -## Changes since 3.2.0: +## Changes in this version: ### BUG FIXES: @@ -23,7 +97,7 @@ Protocol: 31 (unchanged) their buggy compiler (since the configure test is apparently not finding all the compilers that will to crash and burn). - - Fix an issue in the md2man script when building from an external dir. + - Fixed an issue in the md2man script when building from an alternate dir. - Disable `--atimes` on macOS (it apparently just ignores the atime change). @@ -67,9 +141,7 @@ Protocol: 31 (unchanged) # NEWS for rsync 3.2.0 (19 Jun 2020) -Protocol: 31 (unchanged) - -## Changes since 3.1.3: +## Changes in this version: ### BUG FIXES: @@ -122,10 +194,17 @@ Protocol: 31 (unchanged) ### ENHANCEMENTS: + - The default systemd config was made stricter by default. For instance, + `ProtectHome=on` (which hides content in /root and /home/USER dirs), + `ProtectSystem=full` (which makes /usr, /boot, & /etc dirs read-only), and + `PrivateDevices=on` (which hides devices). You can override any of these + using the standard `systemctl edit rsync` and add one or more directives + under a `[Service]` heading (and restart the rsync service). + - Various checksum enhancements, including the optional use of openssl's MD4 & MD5 checksum algorithms, some x86-64 optimizations for the rolling checksum, some x86-64 optimizations for the (non-openssl) MD5 checksum, the addition - of xxhash checksum support, and a negotiation heuristic that ensures that it + of xxHash checksum support, and a negotiation heuristic that ensures that it is easier to add new checksum algorithms in the future. The environment variable `RSYNC_CHECKSUM_LIST` can be used to customize the preference order of the negotiation, or use `--checksum-choice` (`--cc`) to force a choice. @@ -274,9 +353,7 @@ Protocol: 31 (unchanged) # NEWS for rsync 3.1.3 (28 Jan 2018) -Protocol: 31 (unchanged) - -## Changes since 3.1.2: +## Changes in this version: ### SECURITY FIXES: @@ -346,9 +423,7 @@ Protocol: 31 (unchanged) # NEWS for rsync 3.1.2 (21 Dec 2015) -Protocol: 31 (unchanged) - -## Changes since 3.1.1: +## Changes in this version: ### SECURITY FIXES: @@ -414,9 +489,7 @@ Protocol: 31 (unchanged) # NEWS for rsync 3.1.1 (22 Jun 2014) -Protocol: 31 (unchanged) - -## Changes since 3.1.0: +## Changes in this version: ### BUG FIXES: @@ -536,9 +609,11 @@ Protocol: 31 (unchanged) # NEWS for rsync 3.1.0 (28 Sep 2013) -Protocol: 31 (changed) +## Changes in this version: -## Changes since 3.0.9: +### PROTOCOL NUMBER: + + - The protocol number was changed to 31. ### OUTPUT CHANGES: @@ -785,9 +860,7 @@ Protocol: 31 (changed) # NEWS for rsync 3.0.9 (23 Sep 2011) -Protocol: 30 (unchanged) - -## Changes since 3.0.8: +## Changes in this version: ### BUG FIXES: @@ -847,9 +920,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.8 (26 Mar 2011) -Protocol: 30 (unchanged) - -## Changes since 3.0.7: +## Changes in this version: ### BUG FIXES: @@ -988,9 +1059,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.7 (31 Dec 2009) -Protocol: 30 (unchanged) - -## Changes since 3.0.6: +## Changes in this version: ### BUG FIXES: @@ -1058,9 +1127,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.6 (8 May 2009) -Protocol: 30 (unchanged) - -## Changes since 3.0.5: +## Changes in this version: ### BUG FIXES: @@ -1119,9 +1186,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.5 (28 Dec 2008) -Protocol: 30 (unchanged) - -## Changes since 3.0.4: +## Changes in this version: ### BUG FIXES: @@ -1186,9 +1251,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.4 (6 Sep 2008) -Protocol: 30 (unchanged) - -## Changes since 3.0.3: +## Changes in this version: ### BUG FIXES: @@ -1256,9 +1319,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.3 (29 Jun 2008) -Protocol: 30 (unchanged) - -## Changes since 3.0.2: +## Changes in this version: ### BUG FIXES: @@ -1349,9 +1410,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.2 (8 Apr 2008) -Protocol: 30 (unchanged) - -## Changes since 3.0.1: +## Changes in this version: ### BUG FIXES: @@ -1373,9 +1432,7 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.1 (3 Apr 2008) -Protocol: 30 (unchanged) - -## Changes since 3.0.0: +## Changes in this version: ### NOTABLE CHANGES IN BEHAVIOR: @@ -1513,9 +1570,11 @@ Protocol: 30 (unchanged) # NEWS for rsync 3.0.0 (1 Mar 2008) -Protocol: 30 (changed) +## Changes in this version: -## Changes since 2.6.9: +### PROTOCOL NUMBER: + + - The protocol number was changed to 30. ### NOTABLE CHANGES IN BEHAVIOR: @@ -1862,9 +1921,7 @@ Protocol: 30 (changed) # NEWS for rsync 2.6.9 (6 Nov 2006) -Protocol: 29 (unchanged) - -## Changes since 2.6.8: +## Changes in this version: ### BUG FIXES: @@ -2024,9 +2081,7 @@ Protocol: 29 (unchanged) # NEWS for rsync 2.6.8 (22 Apr 2006) -Protocol: 29 (unchanged) - -## Changes since 2.6.7: +## Changes in this version: ### BUG FIXES: @@ -2096,9 +2151,7 @@ Protocol: 29 (unchanged) # NEWS for rsync 2.6.7 (11 Mar 2006) -Protocol: 29 (unchanged) - -## Changes since 2.6.6: +## Changes in this version: ### OUTPUT CHANGES: @@ -2422,9 +2475,7 @@ Protocol: 29 (unchanged) # NEWS for rsync 2.6.6 (28 Jul 2005) -Protocol: 29 (unchanged) - -## Changes since 2.6.5: +## Changes in this version: ### SECURITY FIXES: @@ -2490,9 +2541,7 @@ Protocol: 29 (unchanged) # NEWS for rsync 2.6.5 (1 Jun 2005) -Protocol: 29 (unchanged) - -## Changes since 2.6.4: +## Changes in this version: ### OUTPUT CHANGES: @@ -2673,9 +2722,11 @@ Protocol: 29 (unchanged) # NEWS for rsync 2.6.4 (30 March 2005) -Protocol: 29 (changed) +## Changes in this version: -## Changes since 2.6.3: +### PROTOCOL NUMBER: + + - The protocol number was changed to 29. ### OUTPUT CHANGES: @@ -3053,9 +3104,7 @@ Protocol: 29 (changed) # NEWS for rsync 2.6.3 (30 Sep 2004) -Protocol: 28 (unchanged) - -## Changes since 2.6.2: +## Changes in this version: ### SECURITY FIXES: @@ -3299,9 +3348,7 @@ Protocol: 28 (unchanged) # NEWS for rsync 2.6.2 (30 Apr 2004) -Protocol: 28 (unchanged) - -## Changes since 2.6.1: +## Changes in this version: ### BUG FIXES: @@ -3343,9 +3390,11 @@ Protocol: 28 (unchanged) # NEWS for rsync 2.6.1 (26 Apr 2004) -Protocol: 28 (changed) +## Changes in this version: -## Changes since 2.6.0: +### PROTOCOL NUMBER: + + - The protocol number was changed to 28. ### SECURITY FIXES: @@ -3533,9 +3582,12 @@ Protocol: 28 (changed) # NEWS for rsync 2.6.0 (1 Jan 2004) -Protocol: 27 (changed) +## Changes in this version: -## Changes since 2.5.7: +### PROTOCOL NUMBER: + + - The protocol number was changed to 27. The maximum accepted protocol number + was increased from 30 to 40. ### ENHANCEMENTS: @@ -3670,9 +3722,7 @@ Protocol: 27 (changed) # NEWS for rsync 2.5.7 (4 Dec 2003) -Protocol: 26 (unchanged) - -## Changes since 2.5.6: +## Changes in this version: ### SECURITY FIXES: @@ -3684,9 +3734,7 @@ Protocol: 26 (unchanged) # NEWS for rsync 2.5.6, aka "the dwd-between-jobs release" (26 Jan 2003) -Protocol: 26 (unchanged) - -## Changes since 2.5.5: +## Changes in this version: ### ENHANCEMENTS: @@ -3781,9 +3829,7 @@ Protocol: 26 (unchanged) # NEWS for rsync 2.5.5, aka Snowy River (2 Apr 2002) -Protocol: 26 (unchanged) - -## Changes since 2.5.4: +## Changes in this version: ### ENHANCEMENTS: @@ -3822,9 +3868,7 @@ Protocol: 26 (unchanged) # NEWS for rsync 2.5.4, aka "Imitation lizard skin" (13 Mar 2002) -Protocol: 26 (unchanged) - -## Changes since 2.5.3: +## Changes in this version: ### BUG FIXES: @@ -3844,9 +3888,7 @@ Protocol: 26 (unchanged) # NEWS for rsync 2.5.3, aka "Happy 26" (11 Mar 2002) -Protocol: 26 (unchanged) - -## Changes since 2.5.2: +## Changes in this version: ### SECURITY FIXES: @@ -3895,9 +3937,7 @@ Protocol: 26 (unchanged) # NEWS for rsync 2.5.2 (26 Jan 2002) -Protocol: 26 (changed) - -## Changes since 2.5.1: +## Changes in this version: ### SECURITY FIXES: @@ -3905,6 +3945,10 @@ Protocol: 26 (changed) some cases we were not sufficiently careful about reading integers from the network. +### PROTOCOL NUMBER: + + - The protocol number was changed to 26. + ### BUG FIXES: - Fix possible string mangling in log files. @@ -3942,9 +3986,7 @@ Protocol: 26 (changed) # NEWS for rsync 2.5.1 (3 Jan 2002) -Protocol: 25 (unchanged) - -## Changes since 2.5.0: +## Changes in this version: ### BUG FIXES: @@ -3979,9 +4021,11 @@ Protocol: 25 (unchanged) # NEWS for rsync 2.5.0 (30 Nov 2001) -Protocol: 25 (changed) +## Changes in this version: -## Changes since 2.4.6: +### PROTOCOL NUMBER: + + - The protocol number was changed to 25. ### ANNOUNCEMENTS: @@ -4099,6 +4143,7 @@ Protocol: 25 (changed) | RELEASE DATE | VER. | DATE OF COMMIT\* | PROTOCOL | |--------------|--------|------------------|-------------| +| 04 Jul 2020 | 3.2.2 | | 31 | | 22 Jun 2020 | 3.2.1 | | 31 | | 19 Jun 2020 | 3.2.0 | | 31 | | 28 Jan 2018 | 3.1.3 | | 31 | diff --git a/access.c b/access.c index 5b662901..d7bf01cc 100644 --- a/access.c +++ b/access.c @@ -19,6 +19,7 @@ */ #include "rsync.h" +#include "ifuncs.h" static int allow_forward_dns; @@ -52,10 +53,8 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t if (strcmp(addr, inet_ntoa(*(struct in_addr*)(hp->h_addr_list[i]))) == 0) { /* If reverse lookups are off, we'll use the conf-specified * hostname in preference to UNDETERMINED. */ - if (host == undetermined_hostname) { - if (!(*host_ptr = strdup(tok))) - *host_ptr = undetermined_hostname; - } + if (host == undetermined_hostname) + *host_ptr = strdup(tok); return 1; } } @@ -241,9 +240,6 @@ static int access_match(const char *list, const char *addr, const char **host_pt char *tok; char *list2 = strdup(list); - if (!list2) - out_of_memory("access_match"); - strlower(list2); for (tok = strtok(list2, " ,\t"); tok; tok = strtok(NULL, " ,\t")) { diff --git a/acls.c b/acls.c index 0b5dec6a..4303c2a7 100644 --- a/acls.c +++ b/acls.c @@ -168,8 +168,6 @@ static rsync_acl *create_racl(void) { rsync_acl *racl = new(rsync_acl); - if (!racl) - out_of_memory("create_racl"); *racl = empty_rsync_acl; return racl; @@ -335,8 +333,7 @@ static BOOL unpack_smb_acl(SMB_ACL_T sacl, rsync_acl *racl) qsort(temp_ida_list.items, temp_ida_list.count, sizeof (id_access), id_access_sorter); } #endif - if (!(racl->names.idas = new_array(id_access, temp_ida_list.count))) - out_of_memory("unpack_smb_acl"); + racl->names.idas = new_array(id_access, temp_ida_list.count); memcpy(racl->names.idas, temp_ida_list.items, temp_ida_list.count * sizeof (id_access)); } else racl->names.idas = NULL; @@ -505,9 +502,7 @@ static int get_rsync_acl(const char *fname, rsync_acl *racl, if (cnt) { char *bp = buf + 4*4; - id_access *ida; - if (!(ida = racl->names.idas = new_array(id_access, cnt))) - out_of_memory("get_rsync_acl"); + id_access *ida = racl->names.idas = new_array(id_access, cnt); racl->names.count = cnt; for ( ; cnt--; ida++, bp += 4+4) { ida->id = IVAL(bp, 0); @@ -703,12 +698,7 @@ static uchar recv_ida_entries(int f, ida_entries *ent) uchar computed_mask_bits = 0; int i, count = read_varint(f); - if (count) { - if (!(ent->idas = new_array(id_access, count))) - out_of_memory("recv_ida_entries"); - } else - ent->idas = NULL; - + ent->idas = count ? new_array(id_access, count) : NULL; ent->count = count; for (i = 0; i < count; i++) { diff --git a/authenticate.c b/authenticate.c index 169331e5..3ef83ef2 100644 --- a/authenticate.c +++ b/authenticate.c @@ -20,6 +20,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" extern int read_only; extern char *password_file; @@ -250,8 +251,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host, } *pass++ = '\0'; - if (!(users = strdup(users))) - out_of_memory("auth_server"); + users = strdup(users); for (tok = strtok(users, " ,\t"); tok; tok = strtok(NULL, " ,\t")) { char *opts; @@ -287,8 +287,7 @@ char *auth_server(int f_in, int f_out, int module, const char *host, else { gid_t *gid_array = gid_list.items; auth_uid_groups_cnt = gid_list.count; - if ((auth_uid_groups = new_array(char *, auth_uid_groups_cnt)) == NULL) - out_of_memory("auth_server"); + auth_uid_groups = new_array(char *, auth_uid_groups_cnt); for (j = 0; j < auth_uid_groups_cnt; j++) auth_uid_groups[j] = gid_to_group(gid_array[j]); } diff --git a/batch.c b/batch.c index 805acc6f..a9711c56 100644 --- a/batch.c +++ b/batch.c @@ -38,13 +38,10 @@ extern int do_compression; extern int inplace; extern int append_mode; extern int write_batch; -extern int xfersum_type; extern int protocol_version; extern int raw_argc, cooked_argc; extern char **raw_argv, **cooked_argv; extern char *batch_name; -extern const char *checksum_choice; -extern const char *compress_choice; #ifdef ICONV_OPTION extern char *iconv_opt; #endif @@ -269,14 +266,6 @@ void write_batch_shell_file(void) err |= write_opt("--exclude-from", "-"); } - /* We need to make sure that any protocol-based or negotiated choices get accurately - * reflected in the options we save AND that we avoid any need for --read-batch to - * do a string-based negotiation (since we don't write them into the file). */ - if (do_compression) - err |= write_opt("--compress-choice", compress_choice); - if (strchr(checksum_choice, ',') || xfersum_type != parse_csum_name(NULL, -1)) - err |= write_opt("--checksum-choice", checksum_choice); - /* Elide the filename args from the option list, but scan for them in reverse. */ for (i = raw_argc-1, j = cooked_argc-1; i > 0 && j >= 0; i--) { if (strcmp(raw_argv[i], cooked_argv[j]) == 0) { diff --git a/checksum.c b/checksum.c index 87e83658..6dabfb08 100644 --- a/checksum.c +++ b/checksum.c @@ -27,8 +27,12 @@ */ #include "rsync.h" + #ifdef SUPPORT_XXHASH #include "xxhash.h" +# if XXH_VERSION_NUMBER >= 800 +# define SUPPORT_XXH3 1 +# endif #endif extern int am_server; @@ -40,6 +44,10 @@ extern const char *checksum_choice; struct name_num_obj valid_checksums = { "checksum", NULL, NULL, 0, 0, { +#ifdef SUPPORT_XXH3 + { CSUM_XXH3_128, "xxh128", NULL }, + { CSUM_XXH3_64, "xxh3", NULL }, +#endif #ifdef SUPPORT_XXHASH { CSUM_XXH64, "xxh64", NULL }, { CSUM_XXH64, "xxhash", NULL }, @@ -85,7 +93,7 @@ static const char *checksum_name(int num) { struct name_num_item *nni = get_nni_by_num(&valid_checksums, num); - return nni ? nni->name : num < CSUM_MD4 ? "MD4" : "UNKNOWN"; + return nni ? nni->name : num < CSUM_MD4 ? "md4" : "UNKNOWN"; } void parse_checksum_choice(int final_call) @@ -99,6 +107,8 @@ void parse_checksum_choice(int final_call) checksum_type = parse_csum_name(cp+1, -1); } else xfersum_type = checksum_type = parse_csum_name(checksum_choice, -1); + if (am_server && checksum_choice) + validate_choice_vs_env(NSTR_CHECKSUM, xfersum_type, checksum_type); } if (xfersum_type == CSUM_NONE) @@ -133,10 +143,11 @@ int csum_len_for_type(int cst, BOOL flist_csum) return MD4_DIGEST_LEN; case CSUM_MD5: return MD5_DIGEST_LEN; -#ifdef SUPPORT_XXHASH case CSUM_XXH64: + case CSUM_XXH3_64: return 64/8; -#endif + case CSUM_XXH3_128: + return 128/8; default: /* paranoia to prevent missing case values */ exit_cleanup(RERR_UNSUPPORTED); } @@ -158,10 +169,10 @@ int canonical_checksum(int csum_type) case CSUM_MD4: case CSUM_MD5: return -1; -#ifdef SUPPORT_XXHASH case CSUM_XXH64: + case CSUM_XXH3_64: + case CSUM_XXH3_128: return 1; -#endif default: /* paranoia to prevent missing case values */ exit_cleanup(RERR_UNSUPPORTED); } @@ -198,6 +209,17 @@ void get_checksum2(char *buf, int32 len, char *sum) case CSUM_XXH64: SIVAL64(sum, 0, XXH64(buf, len, checksum_seed)); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + SIVAL64(sum, 0, XXH3_64bits_withSeed(buf, len, checksum_seed)); + break; + case CSUM_XXH3_128: { + XXH128_hash_t digest = XXH3_128bits_withSeed(buf, len, checksum_seed); + SIVAL64(sum, 0, digest.low64); + SIVAL64(sum, 8, digest.high64); + break; + } #endif case CSUM_MD5: { MD5_CTX m5; @@ -249,8 +271,6 @@ void get_checksum2(char *buf, int32 len, char *sum) free(buf1); buf1 = new_array(char, len+4); len1 = len; - if (!buf1) - out_of_memory("get_checksum2"); } memcpy(buf1, buf, len); @@ -313,6 +333,45 @@ void file_checksum(const char *fname, const STRUCT_STAT *st_p, char *sum) SIVAL64(sum, 0, XXH64_digest(state)); break; } +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: { + static XXH3_state_t* state = NULL; + if (!state && !(state = XXH3_createState())) + out_of_memory("file_checksum"); + + XXH3_64bits_reset(state); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); + + remainder = (int32)(len - i); + if (remainder > 0) + XXH3_64bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); + + SIVAL64(sum, 0, XXH3_64bits_digest(state)); + break; + } + case CSUM_XXH3_128: { + XXH128_hash_t digest; + static XXH3_state_t* state = NULL; + if (!state && !(state = XXH3_createState())) + out_of_memory("file_checksum"); + + XXH3_128bits_reset(state); + + for (i = 0; i + CHUNK_SIZE <= len; i += CHUNK_SIZE) + XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, CHUNK_SIZE), CHUNK_SIZE); + + remainder = (int32)(len - i); + if (remainder > 0) + XXH3_128bits_update(state, (uchar *)map_ptr(buf, i, remainder), remainder); + + digest = XXH3_128bits_digest(state); + SIVAL64(sum, 0, digest.low64); + SIVAL64(sum, 8, digest.high64); + break; + } #endif case CSUM_MD5: { MD5_CTX m5; @@ -389,6 +448,9 @@ static union { #ifdef SUPPORT_XXHASH static XXH64_state_t* xxh64_state; #endif +#ifdef SUPPORT_XXH3 +static XXH3_state_t* xxh3_state; +#endif static int cursum_type; void sum_init(int csum_type, int seed) @@ -406,6 +468,18 @@ void sum_init(int csum_type, int seed) out_of_memory("sum_init"); XXH64_reset(xxh64_state, 0); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + if (!xxh3_state && !(xxh3_state = XXH3_createState())) + out_of_memory("sum_init"); + XXH3_64bits_reset(xxh3_state); + break; + case CSUM_XXH3_128: + if (!xxh3_state && !(xxh3_state = XXH3_createState())) + out_of_memory("sum_init"); + XXH3_128bits_reset(xxh3_state); + break; #endif case CSUM_MD5: MD5_Init(&ctx.m5); @@ -448,6 +522,14 @@ void sum_update(const char *p, int32 len) case CSUM_XXH64: XXH64_update(xxh64_state, p, len); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + XXH3_64bits_update(xxh3_state, p, len); + break; + case CSUM_XXH3_128: + XXH3_128bits_update(xxh3_state, p, len); + break; #endif case CSUM_MD5: MD5_Update(&ctx.m5, (uchar *)p, len); @@ -502,6 +584,17 @@ int sum_end(char *sum) case CSUM_XXH64: SIVAL64(sum, 0, XXH64_digest(xxh64_state)); break; +#endif +#ifdef SUPPORT_XXH3 + case CSUM_XXH3_64: + SIVAL64(sum, 0, XXH3_64bits_digest(xxh3_state)); + break; + case CSUM_XXH3_128: { + XXH128_hash_t digest = XXH3_128bits_digest(xxh3_state); + SIVAL64(sum, 0, digest.low64); + SIVAL64(sum, 8, digest.high64); + break; + } #endif case CSUM_MD5: MD5_Final((uchar *)sum, &ctx.m5); diff --git a/cleanup.c b/cleanup.c index cf8330ac..a2d6b384 100644 --- a/cleanup.c +++ b/cleanup.c @@ -221,8 +221,13 @@ NORETURN void _exit_cleanup(int code, const char *file, int line) /* If line < 0, this exit is after a MSG_ERROR_EXIT event, so * we don't want to output a duplicate error. */ if ((exit_code && line > 0) - || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1)))) + || am_daemon || (logfile_name && (am_server || !INFO_GTE(STATS, 1)))) { +#ifdef HAVE_USLEEP /* A tiny delay just in case both sender & receiver are sending a msg at the same time. */ + if (am_server && exit_code) + usleep(50); +#endif log_exit(exit_code, exit_file, exit_line); + } #include "case_N.h" switch_step++; diff --git a/clientserver.c b/clientserver.c index b790974c..831c3eab 100644 --- a/clientserver.c +++ b/clientserver.c @@ -38,6 +38,7 @@ extern int preserve_xattrs; extern int kluge_around_eof; extern int daemon_over_rsh; extern int munge_symlinks; +extern int open_noatime; extern int sanitize_paths; extern int numeric_ids; extern int filesfrom_fd; @@ -235,8 +236,7 @@ int start_inband_exchange(int f_in, int f_out, const char *user, int argc, char else modlen = p - *argv; - if (!(modname = new_array(char, modlen+1+1))) /* room for '/' & '\0' */ - out_of_memory("start_inband_exchange"); + modname = new_array(char, modlen+1+1); /* room for '/' & '\0' */ strlcpy(modname, *argv, modlen + 1); modname[modlen] = '/'; modname[modlen+1] = '\0'; @@ -696,8 +696,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char module_id = i; - if (lp_transfer_logging(i) && !logfile_format) - logfile_format = lp_log_format(i); + if (lp_transfer_logging(module_id) && !logfile_format) + logfile_format = lp_log_format(module_id); if (log_format_has(logfile_format, 'i')) logfile_format_has_i = 1; if (logfile_format_has_i || log_format_has(logfile_format, 'o')) @@ -706,7 +706,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char uid = MY_UID(); am_root = (uid == 0); - p = *lp_uid(i) ? lp_uid(i) : am_root ? NOBODY_USER : NULL; + p = *lp_uid(module_id) ? lp_uid(module_id) : am_root ? NOBODY_USER : NULL; if (p) { if (!user_to_uid(p, &uid, True)) { rprintf(FLOG, "Invalid uid %s\n", p); @@ -717,7 +717,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char } else set_uid = 0; - p = *lp_gid(i) ? conf_strtok(lp_gid(i)) : NULL; + p = *lp_gid(module_id) ? conf_strtok(lp_gid(module_id)) : NULL; if (p) { /* The "*" gid must be the first item in the list. */ if (strcmp(p, "*") == 0) { @@ -750,7 +750,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char return -1; } - module_dir = lp_path(i); + module_dir = lp_path(module_id); if (*module_dir == '\0') { rprintf(FLOG, "No path specified for module %s\n", name); io_printf(f_out, "@ERROR: no path setting.\n"); @@ -787,38 +787,38 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char } else set_filter_dir(module_dir, module_dirlen); - p = lp_filter(i); + p = lp_filter(module_id); parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT), XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3); - p = lp_include_from(i); + p = lp_include_from(module_id); parse_filter_file(&daemon_filter_list, p, rule_template(FILTRULE_INCLUDE), XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS); - p = lp_include(i); + p = lp_include(module_id); parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_INCLUDE | FILTRULE_WORD_SPLIT), XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES); - p = lp_exclude_from(i); + p = lp_exclude_from(module_id); parse_filter_file(&daemon_filter_list, p, rule_template(0), XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES | XFLG_FATAL_ERRORS); - p = lp_exclude(i); + p = lp_exclude(module_id); parse_filter_str(&daemon_filter_list, p, rule_template(FILTRULE_WORD_SPLIT), XFLG_ABS_IF_SLASH | XFLG_DIR2WILD3 | XFLG_OLD_PREFIXES); log_init(1); #ifdef HAVE_PUTENV - if ((*lp_early_exec(i) || *lp_prexfer_exec(i) || *lp_postxfer_exec(i)) + if ((*lp_early_exec(module_id) || *lp_prexfer_exec(module_id) || *lp_postxfer_exec(module_id)) && !getenv("RSYNC_NO_XFER_EXEC")) { set_env_num("RSYNC_PID", (long)getpid()); /* For post-xfer exec, fork a new process to run the rsync * daemon while this process waits for the exit status and * runs the indicated command at that point. */ - if (*lp_postxfer_exec(i)) { + if (*lp_postxfer_exec(module_id)) { pid_t pid = fork(); if (pid < 0) { rsyserr(FLOG, errno, "fork failed"); @@ -838,7 +838,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char else status = -1; set_env_num("RSYNC_EXIT_STATUS", status); - if (shell_exec(lp_postxfer_exec(i)) < 0) + if (shell_exec(lp_postxfer_exec(module_id)) < 0) status = -1; _exit(status); } @@ -846,9 +846,9 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char /* For early exec, fork a child process to run the indicated * command and wait for it to exit. */ - if (*lp_early_exec(i)) { + if (*lp_early_exec(module_id)) { int arg_fd; - pid_t pid = start_pre_exec(lp_early_exec(i), &arg_fd, NULL); + pid_t pid = start_pre_exec(lp_early_exec(module_id), &arg_fd, NULL); if (pid == (pid_t)-1) { rsyserr(FLOG, errno, "early exec preparation failed"); io_printf(f_out, "@ERROR: early exec preparation failed\n"); @@ -865,8 +865,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char /* For pre-xfer exec, fork a child process to run the indicated * command, though it first waits for the parent process to * send us the user's request via a pipe. */ - if (*lp_prexfer_exec(i)) { - pre_exec_pid = start_pre_exec(lp_prexfer_exec(i), &pre_exec_arg_fd, &pre_exec_error_fd); + if (*lp_prexfer_exec(module_id)) { + pre_exec_pid = start_pre_exec(lp_prexfer_exec(module_id), &pre_exec_arg_fd, &pre_exec_error_fd); if (pre_exec_pid == (pid_t)-1) { rsyserr(FLOG, errno, "pre-xfer exec preparation failed"); io_printf(f_out, "@ERROR: pre-xfer exec preparation failed\n"); @@ -907,7 +907,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char if (module_dirlen || (!use_chroot && !*lp_daemon_chroot())) sanitize_paths = 1; - if ((munge_symlinks = lp_munge_symlinks(i)) < 0) + if ((munge_symlinks = lp_munge_symlinks(module_id)) < 0) munge_symlinks = !use_chroot || module_dirlen; if (munge_symlinks) { STRUCT_STAT st; @@ -962,8 +962,8 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char am_root = (our_uid == 0); } - if (lp_temp_dir(i) && *lp_temp_dir(i)) { - tmpdir = lp_temp_dir(i); + if (lp_temp_dir(module_id) && *lp_temp_dir(module_id)) { + tmpdir = lp_temp_dir(module_id); if (strlen(tmpdir) >= MAXPATHLEN - 10) { rprintf(FLOG, "the 'temp dir' value for %s is WAY too long -- ignoring.\n", @@ -990,7 +990,12 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char } else orig_early_argv = NULL; + /* The default is to use the user's setting unless the module sets True or False. */ + if (lp_open_noatime(module_id) >= 0) + open_noatime = lp_open_noatime(module_id); + munge_symlinks = save_munge_symlinks; /* The client mustn't control this. */ + if (am_daemon > 0) msgs2stderr = 0; /* A non-rsh-run daemon doesn't have stderr for msgs. */ @@ -1009,7 +1014,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char if (write_batch < 0) dry_run = 1; - if (lp_fake_super(i)) { + if (lp_fake_super(module_id)) { if (preserve_xattrs > 1) preserve_xattrs = 1; am_root = -1; @@ -1034,7 +1039,7 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char #ifndef DEBUG /* don't allow the logs to be flooded too fast */ - limit_output_verbosity(lp_max_verbosity(i)); + limit_output_verbosity(lp_max_verbosity(module_id)); #endif if (protocol_version < 23 && (protocol_version == 22 || am_sender)) @@ -1095,20 +1100,20 @@ static int rsync_module(int f_in, int f_out, int i, const char *addr, const char #endif if (!numeric_ids - && (use_chroot ? lp_numeric_ids(i) != False : lp_numeric_ids(i) == True)) + && (use_chroot ? lp_numeric_ids(module_id) != False : lp_numeric_ids(module_id) == True)) numeric_ids = -1; /* Set --numeric-ids w/o breaking protocol. */ - if (lp_timeout(i) && (!io_timeout || lp_timeout(i) < io_timeout)) - set_io_timeout(lp_timeout(i)); + if (lp_timeout(module_id) && (!io_timeout || lp_timeout(module_id) < io_timeout)) + set_io_timeout(lp_timeout(module_id)); /* If we have some incoming/outgoing chmod changes, append them to * any user-specified changes (making our changes have priority). * We also get a pointer to just our changes so that a receiver * process can use them separately if --perms wasn't specified. */ if (am_sender) - p = lp_outgoing_chmod(i); + p = lp_outgoing_chmod(module_id); else - p = lp_incoming_chmod(i); + p = lp_incoming_chmod(module_id); if (*p && !(daemon_chmod_modes = parse_chmod(p, &chmod_modes))) { rprintf(FLOG, "Invalid \"%sing chmod\" directive: %s\n", am_sender ? "outgo" : "incom", p); @@ -1233,8 +1238,7 @@ int start_daemon(int f_in, int f_out) io_printf(f_out, "@ERROR: invalid early_input length\n"); return -1; } - if (!(early_input = new_array(char, early_input_len))) - out_of_memory("exchange_protocols"); + early_input = new_array(char, early_input_len); read_buf(f_in, early_input, early_input_len); if (!read_line_old(f_in, line, sizeof line, 0)) diff --git a/compat.c b/compat.c index 9ec39ea0..527201ac 100644 --- a/compat.c +++ b/compat.c @@ -20,6 +20,7 @@ */ #include "rsync.h" +#include "itypes.h" extern int am_server; extern int am_sender; @@ -173,6 +174,8 @@ void parse_compress_choice(int final_call) exit_cleanup(RERR_UNSUPPORTED); } do_compression = nni->num; + if (am_server) + validate_choice_vs_env(NSTR_COMPRESS, do_compression, -1); } else if (do_compression) do_compression = CPRES_ZLIB; else @@ -241,8 +244,7 @@ static void init_nno_saw(struct name_num_obj *nno, int val) } if (!nno->saw) { - if (!(nno->saw = new_array0(uchar, nno->saw_len))) - out_of_memory("init_nno_saw"); + nno->saw = new_array0(uchar, nno->saw_len); /* We'll take this opportunity to make sure that the main_name values are set right. */ for (cnt = 1, nni = nno->list; nni->name; nni++, cnt++) { @@ -261,10 +263,14 @@ static void init_nno_saw(struct name_num_obj *nno, int val) static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf, int tobuf_len) { char *to = tobuf, *tok = NULL; - int cnt = 0; + int saw_tok = 0, cnt = 0; while (1) { - if (*from == ' ' || !*from) { + int at_space = isSpace(from); + char ch = *from++; + if (ch == '&') + ch = '\0'; + if (!ch || at_space) { if (tok) { struct name_num_item *nni = get_nni_by_name(nno, tok, to - tok); if (nni && !nno->saw[nni->num]) { @@ -278,9 +284,10 @@ static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf } } else to = tok - (tok != tobuf); + saw_tok = 1; tok = NULL; } - if (!*from++) + if (!ch) break; continue; } @@ -293,13 +300,19 @@ static int parse_nni_str(struct name_num_obj *nno, const char *from, char *tobuf to = tok - (tok != tobuf); break; } - *to++ = *from++; + *to++ = ch; } *to = '\0'; + if (saw_tok && to == tobuf) + return strlcpy(tobuf, "INVALID", MAX_NSTR_STRLEN); + return to - tobuf; } +/* This routine is always called with a tmpbuf of MAX_NSTR_STRLEN length, but the + * buffer may be pre-populated with a "len" length string to use OR a len of -1 + * to tell us to read a string from the fd. */ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, int len) { struct name_num_item *ret = NULL; @@ -315,17 +328,26 @@ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, } if (len > 0) { + struct name_num_item *nni; int best = nno->saw_len; /* We want best == 1 from the client list, so start with a big number. */ - char *tok; - if (am_server) - init_nno_saw(nno, 1); /* Since we're parsing client names, anything we parse first is #1. */ - for (tok = strtok(tmpbuf, " \t"); tok; tok = strtok(NULL, " \t")) { - struct name_num_item *nni = get_nni_by_name(nno, tok, -1); + char *space, *tok = tmpbuf; + while (tok) { + while (*tok == ' ') tok++; /* Should be unneeded... */ + if (!*tok) + break; + if ((space = strchr(tok, ' ')) != NULL) + *space = '\0'; + nni = get_nni_by_name(nno, tok, -1); + if (space) { + *space = ' '; + tok = space + 1; + } else + tok = NULL; if (!nni || !nno->saw[nni->num] || best <= nno->saw[nni->num]) continue; ret = nni; best = nno->saw[nni->num]; - if (best == 1) + if (best == 1 || am_server) /* The server side stops at the first acceptable client choice */ break; } if (ret) { @@ -337,15 +359,84 @@ static void recv_negotiate_str(int f_in, struct name_num_obj *nno, char *tmpbuf, } } - if (!am_server) - rprintf(FERROR, "Failed to negotiate a common %s\n", nno->type); + if (!am_server || !do_negotiated_strings) { + char *cp = tmpbuf; + int j; + rprintf(FERROR, "Failed to negotiate a %s choice.\n", nno->type); + rprintf(FERROR, "%s list: %s\n", am_server ? "Client" : "Server", tmpbuf); + /* Recreate our original list from the saw values. This can't overflow our huge + * buffer because we don't have enough valid entries to get anywhere close. */ + for (j = 1, *cp = '\0'; j <= nno->saw_len; j++) { + struct name_num_item *nni; + for (nni = nno->list; nni->name; nni++) { + if (nno->saw[nni->num] == j) { + *cp++ = ' '; + cp += strlcpy(cp, nni->name, MAX_NSTR_STRLEN - (cp - tmpbuf)); + break; + } + } + } + if (!*tmpbuf) + strlcpy(cp, " INVALID", MAX_NSTR_STRLEN); + rprintf(FERROR, "%s list:%s\n", am_server ? "Server" : "Client", tmpbuf); + } + exit_cleanup(RERR_UNSUPPORTED); } +static const char *getenv_nstr(int ntype) +{ + const char *env_str = getenv(ntype == NSTR_COMPRESS ? "RSYNC_COMPRESS_LIST" : "RSYNC_CHECKSUM_LIST"); + + /* When writing a batch file, we always negotiate an old-style choice. */ + if (write_batch) + env_str = ntype == NSTR_COMPRESS ? "zlib" : protocol_version >= 30 ? "md5" : "md4"; + + if (am_server && env_str) { + char *cp = strchr(env_str, '&'); + if (cp) + env_str = cp + 1; + } + + return env_str; +} + +void validate_choice_vs_env(int ntype, int num1, int num2) +{ + struct name_num_obj *nno = ntype == NSTR_COMPRESS ? &valid_compressions : &valid_checksums; + const char *list_str = getenv_nstr(ntype); + char tmpbuf[MAX_NSTR_STRLEN]; + + if (!list_str) + return; + + while (isSpace(list_str)) list_str++; + + if (!*list_str) + return; + + init_nno_saw(nno, 0); + parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN); + + if (ntype == NSTR_CHECKSUM) /* If "md4" is in the env list, all the old MD4 choices are OK too. */ + nno->saw[CSUM_MD4_ARCHAIC] = nno->saw[CSUM_MD4_BUSTED] = nno->saw[CSUM_MD4_OLD] = nno->saw[CSUM_MD4]; + + if (!nno->saw[num1] || (num2 >= 0 && !nno->saw[num2])) { + rprintf(FERROR, "Your --%s-choice value (%s) was refused by the server.\n", + ntype == NSTR_COMPRESS ? "compress" : "checksum", + ntype == NSTR_COMPRESS ? compress_choice : checksum_choice); + exit_cleanup(RERR_UNSUPPORTED); + } + + free(nno->saw); + nno->saw = NULL; +} + /* The saw buffer is initialized and used to store ordinal values from 1 to N * for the order of the args in the array. If dup_markup == '\0', duplicates * are removed otherwise the char is prefixed to the duplicate term and, if it - * is an opening paren/bracket/brace, the matching closing char is suffixed. */ + * is an opening paren/bracket/brace, the matching closing char is suffixed. + * "none" is removed on the client side unless dup_markup != '\0'. */ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup) { struct name_num_item *nni; @@ -367,6 +458,8 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, continue; delim = dup_markup; } + if (nni->num == 0 && !am_server && !dup_markup) + continue; if (len) to_buf[len++]= ' '; if (delim) { @@ -386,25 +479,15 @@ int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, return len; } -static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char *env_name) +static void send_negotiate_str(int f_out, struct name_num_obj *nno, int ntype) { char tmpbuf[MAX_NSTR_STRLEN]; - const char *list_str = getenv(env_name); - int len, fail_if_empty = list_str && strstr(list_str, "FAIL"); + const char *list_str = getenv_nstr(ntype); + int len; - if (!do_negotiated_strings) { - if (!am_server && fail_if_empty) { - rprintf(FERROR, "Remote rsync is too old for %s negotiation\n", nno->type); - exit_cleanup(RERR_UNSUPPORTED); - } - return; - } - - if (list_str && *list_str && (!am_server || local_server)) { + if (list_str && *list_str) { init_nno_saw(nno, 0); len = parse_nni_str(nno, list_str, tmpbuf, MAX_NSTR_STRLEN); - if (fail_if_empty && !len) - len = strlcpy(tmpbuf, "FAIL", MAX_NSTR_STRLEN); list_str = tmpbuf; } else list_str = NULL; @@ -419,15 +502,10 @@ static void send_negotiate_str(int f_out, struct name_num_obj *nno, const char * rprintf(FINFO, "Client %s list (on client): %s\n", nno->type, tmpbuf); } - if (local_server) { - /* A local server doesn't bother to send/recv the strings, it just constructs - * and parses the same string on both sides. */ - recv_negotiate_str(-1, nno, tmpbuf, len); - } else { - /* Each side sends their list of valid names to the other side and then both sides - * pick the first name in the client's list that is also in the server's list. */ + /* Each side sends their list of valid names to the other side and then both sides + * pick the first name in the client's list that is also in the server's list. */ + if (do_negotiated_strings) write_vstring(f_out, tmpbuf, len); - } } static void negotiate_the_strings(int f_in, int f_out) @@ -435,20 +513,35 @@ static void negotiate_the_strings(int f_in, int f_out) /* We send all the negotiation strings before we start to read them to help avoid a slow startup. */ if (!checksum_choice) - send_negotiate_str(f_out, &valid_checksums, "RSYNC_CHECKSUM_LIST"); + send_negotiate_str(f_out, &valid_checksums, NSTR_CHECKSUM); if (do_compression && !compress_choice) - send_negotiate_str(f_out, &valid_compressions, "RSYNC_COMPRESS_LIST"); + send_negotiate_str(f_out, &valid_compressions, NSTR_COMPRESS); if (valid_checksums.saw) { char tmpbuf[MAX_NSTR_STRLEN]; - recv_negotiate_str(f_in, &valid_checksums, tmpbuf, -1); + int len; + if (do_negotiated_strings) + len = -1; + else + len = strlcpy(tmpbuf, protocol_version >= 30 ? "md5" : "md4", MAX_NSTR_STRLEN); + recv_negotiate_str(f_in, &valid_checksums, tmpbuf, len); } if (valid_compressions.saw) { char tmpbuf[MAX_NSTR_STRLEN]; - recv_negotiate_str(f_in, &valid_compressions, tmpbuf, -1); + int len; + if (do_negotiated_strings) + len = -1; + else + len = strlcpy(tmpbuf, "zlib", MAX_NSTR_STRLEN); + recv_negotiate_str(f_in, &valid_compressions, tmpbuf, len); } + + /* If the other side is too old to negotiate, the above steps just made sure that + * the env didn't disallow the old algorithm. Mark things as non-negotiated. */ + if (!do_negotiated_strings) + valid_checksums.negotiated_name = valid_compressions.negotiated_name = NULL; } void setup_protocol(int f_out,int f_in) @@ -602,10 +695,8 @@ void setup_protocol(int f_out,int f_in) if (local_server || strchr(client_info, 'I') != NULL) compat_flags |= CF_INPLACE_PARTIAL_DIR; if (local_server || strchr(client_info, 'v') != NULL) { - if (!write_batch || protocol_version >= 30) { - do_negotiated_strings = 1; - compat_flags |= CF_VARINT_FLIST_FLAGS; - } + do_negotiated_strings = 1; + compat_flags |= CF_VARINT_FLIST_FLAGS; } if (strchr(client_info, 'V') != NULL) { /* Support a pre-release 'V' that got superseded */ if (!write_batch) @@ -654,6 +745,9 @@ void setup_protocol(int f_out,int f_in) #endif } + if (read_batch) + do_negotiated_strings = 0; + if (need_unsorted_flist && (!am_sender || inc_recurse)) unsort_ndx = ++file_extra_cnt; diff --git a/configure b/configure index 32cc3b5d..51c3fee5 100755 --- a/configure +++ b/configure @@ -4,24 +4,24 @@ # then transfer control to the configure.sh script to do the real work. dir=`dirname $0` -realconfigure="$dir/configure.sh" +if test x"$dir" = x; then + dir=. +fi -if test ! -f "$realconfigure"; then - if test -f "$HOME/build_farm/build_test.fns"; then - # Test the included popt - set -- --with-included-popt "${@}" - # Allow the build farm to grab latest files via rsync. - actions='build fetch' - else - actions='build' +if test "$dir" = '.'; then + branch=`packaging/prep-auto-dir` || exit 1 + if test x"$branch" != x; then + cd build || exit 1 + dir=.. fi - if "$dir/prepare-source" $actions; then - : - else +fi + +if test ! -f configure.sh; then + if ! "$dir/prepare-source" build; then echo 'Failed to build configure.sh and/or config.h.in -- giving up.' >&2 - rm -f "$realconfigure" + rm -f configure.sh exit 1 fi fi -exec "$realconfigure" "${@}" +exec ./configure.sh --srcdir="$dir" "${@}" diff --git a/configure.ac b/configure.ac index 28865ea9..112367cb 100644 --- a/configure.ac +++ b/configure.ac @@ -1,17 +1,16 @@ dnl Process this file with autoconf to produce a configure script. -AC_INIT([rsync],[3.2.1],[https://rsync.samba.org/bugtracking.html]) +AC_INIT([rsync],[ ],[https://rsync.samba.org/bugtracking.html]) AC_CONFIG_MACRO_DIR([m4]) AC_CONFIG_SRCDIR([byteorder.h]) AC_CONFIG_HEADER(config.h) AC_PREREQ([2.69]) -AC_SUBST(RSYNC_VERSION, $PACKAGE_VERSION) +PACKAGE_VERSION=`sed 's/.*"\(.*\)".*/\1/' <$srcdir/version.h` + AC_MSG_NOTICE([Configuring rsync $PACKAGE_VERSION]) -AC_DEFINE_UNQUOTED(RSYNC_VERSION, ["$PACKAGE_VERSION"], [rsync release version]) - LDFLAGS=${LDFLAGS-""} AC_CANONICAL_HOST @@ -1338,5 +1337,5 @@ AC_CONFIG_FILES([Makefile lib/dummy zlib/dummy popt/dummy shconfig]) AC_OUTPUT AC_MSG_RESULT() -AC_MSG_RESULT([ rsync ${RSYNC_VERSION} configuration successful]) +AC_MSG_RESULT([ rsync $PACKAGE_VERSION configuration successful]) AC_MSG_RESULT() diff --git a/daemon-parm.awk b/daemon-parm.awk new file mode 100755 index 00000000..ad52e9d9 --- /dev/null +++ b/daemon-parm.awk @@ -0,0 +1,114 @@ +#!/usr/bin/awk -f + +# The caller must pass arg: daemon-parm.txt +# The resulting code is output into daemon-parm.h + +BEGIN { + heading = "/* DO NOT EDIT THIS FILE! It is auto-generated from a list of values in " ARGV[1] "! */\n\n" + sect = psect = defines = accessors = prior_ptype = "" + parms = "\nstatic struct parm_struct parm_table[] = {" + comment_fmt = "\n/********** %s **********/\n" + tdstruct = "typedef struct {" +} + +/^\s*$/ { next } +/^#/ { next } + +/^Globals:/ { + if (defines != "") { + print "The Globals section must come first!" + defines = "" + exit + } + defines = tdstruct + values = "\nstatic const all_vars Defaults = {\n { /* Globals: */\n" + exps = exp_values = sprintf(comment_fmt, "EXP") + sect = "GLOBAL" + psect = ", P_GLOBAL, &Vars.g." + next +} + +/^Locals:/ { + if (sect == "") { + print "The Locals section must come after the Globals!" + exit + } + defines = defines exps "} global_vars;\n\n" tdstruct + values = values exp_values "\n }, { /* Locals: */\n" + exps = exp_values = sprintf(comment_fmt, "EXP") + sect = "LOCAL" + psect = ", P_LOCAL, &Vars.l." + next +} + +/^(STRING|CHAR|PATH|INTEGER|ENUM|OCTAL|BOOL|BOOLREV|BOOL3)[ \t]/ { + ptype = $1 + name = $2 + $1 = $2 = "" + sub(/^[ \t]+/, "") + + if (ptype != prior_ptype) { + comment = sprintf(comment_fmt, ptype) + defines = defines comment + values = values comment + parms = parms "\n" + accessors = accessors "\n" + prior_ptype = ptype + } + + if (ptype == "STRING" || ptype == "PATH") { + atype = "STRING" + vtype = "char*" + } else if (ptype ~ /BOOL/) { + atype = vtype = "BOOL" + } else if (ptype == "CHAR") { + atype = "CHAR" + vtype = "char" + } else { + atype = "INTEGER" + vtype = "int" + } + + # The name might be var_name|public_name + pubname = name + sub(/\|.*/, "", name) + sub(/.*\|/, "", pubname) + gsub(/_/, " ", pubname) + gsub(/-/, "", name) + + if (ptype == "ENUM") + enum = "enum_" name + else + enum = "NULL" + + defines = defines "\t" vtype " " name ";\n" + values = values "\t" $0 ", /* " name " */\n" + parms = parms " {\"" pubname "\", P_" ptype psect name ", " enum ", 0},\n" + accessors = accessors "FN_" sect "_" atype "(lp_" name ", " name ")\n" + + if (vtype == "char*") { + exps = exps "\tBOOL " name "_EXP;\n" + exp_values = exp_values "\tFalse, /* " name "_EXP */\n" + } + + next +} + +/./ { + print "Extraneous line:" $0 + defines = "" + exit +} + +END { + if (sect != "" && defines != "") { + defines = defines exps "} local_vars;\n\n" + defines = defines tdstruct "\n\tglobal_vars g;\n\tlocal_vars l;\n} all_vars;\n" + values = values exp_values "\n }\n};\n\nstatic all_vars Vars;\n" + parms = parms "\n {NULL, P_BOOL, P_NONE, NULL, NULL, 0}\n};\n" + print heading defines values parms accessors > "daemon-parm.h" + } else { + print "Failed to parse the data in " ARGV[1] + exit 1 + } +} diff --git a/daemon-parm.txt b/daemon-parm.txt new file mode 100644 index 00000000..0f8f01e5 --- /dev/null +++ b/daemon-parm.txt @@ -0,0 +1,67 @@ +Globals: ================================================================ + +STRING bind_address|address NULL +STRING daemon_chroot NULL +STRING daemon_gid NULL +STRING daemon_uid NULL +STRING motd_file NULL +STRING pid_file NULL +STRING socket_options NULL + +INTEGER listen_backlog 5 +INTEGER rsync_port|port 0 + +BOOL proxy_protocol False + +Locals: ================================================================= + +STRING auth_users NULL +STRING charset NULL +STRING comment NULL +STRING dont_compress DEFAULT_DONT_COMPRESS +STRING early_exec NULL +STRING exclude NULL +STRING exclude_from NULL +STRING filter NULL +STRING gid NULL +STRING hosts_allow NULL +STRING hosts_deny NULL +STRING include NULL +STRING include_from NULL +STRING incoming_chmod NULL +STRING lock_file DEFAULT_LOCK_FILE +STRING log_file NULL +STRING log_format "%o %h [%a] %m (%u) %f %l" +STRING name NULL +STRING outgoing_chmod NULL +STRING post-xfer_exec NULL +STRING pre-xfer_exec NULL +STRING refuse_options NULL +STRING secrets_file NULL +STRING syslog_tag "rsyncd" +STRING uid NULL + +PATH path NULL +PATH temp_dir NULL + +INTEGER max_connections 0 +INTEGER max_verbosity 1 +INTEGER timeout 0 + +ENUM syslog_facility LOG_DAEMON + +BOOL fake_super False +BOOL forward_lookup True +BOOL ignore_errors False +BOOL ignore_nonreadable False +BOOL list True +BOOL read_only True +BOOL reverse_lookup True +BOOL strict_modes True +BOOL transfer_logging False +BOOL use_chroot True +BOOL write_only False + +BOOL3 munge_symlinks Unset +BOOL3 numeric_ids Unset +BOOL3 open_noatime Unset diff --git a/exclude.c b/exclude.c index df56e802..10b56e76 100644 --- a/exclude.c +++ b/exclude.c @@ -22,6 +22,7 @@ #include "rsync.h" #include "default-cvsignore.h" +#include "ifuncs.h" extern int am_server; extern int am_sender; @@ -200,8 +201,7 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_ } else suf_len = 0; - if (!(rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1))) - out_of_memory("add_rule"); + rule->pattern = new_array(char, pre_len + pat_len + suf_len + 1); if (pre_len) { memcpy(rule->pattern, dirbuf + module_dirlen, pre_len); for (cp = rule->pattern; cp < rule->pattern + pre_len; cp++) { @@ -262,19 +262,14 @@ static void add_rule(filter_rule_list *listp, const char *pat, unsigned int pat_ } } - if (!(lp = new_array0(filter_rule_list, 1))) - out_of_memory("add_rule"); + lp = new_array0(filter_rule_list, 1); if (asprintf(&lp->debug_type, " [per-dir %s]", cp) < 0) out_of_memory("add_rule"); rule->u.mergelist = lp; if (mergelist_cnt == mergelist_size) { mergelist_size += 5; - mergelist_parents = realloc_array(mergelist_parents, - filter_rule *, - mergelist_size); - if (!mergelist_parents) - out_of_memory("add_rule"); + mergelist_parents = realloc_array(mergelist_parents, filter_rule *, mergelist_size); } if (DEBUG_GTE(FILTER, 2)) { rprintf(FINFO, "[%s] activating mergelist #%d%s\n", @@ -498,8 +493,6 @@ void *push_local_filters(const char *dir, unsigned int dirlen) push = (struct local_filter_state *)new_array(char, sizeof (struct local_filter_state) + (mergelist_cnt-1) * sizeof (filter_rule_list)); - if (!push) - out_of_memory("push_local_filters"); push->mergelist_cnt = mergelist_cnt; for (i = 0; i < mergelist_cnt; i++) { @@ -822,8 +815,7 @@ static filter_rule *parse_rule_tok(const char **rulestr_ptr, if (!*s) return NULL; - if (!(rule = new0(filter_rule))) - out_of_memory("parse_rule_tok"); + rule = new0(filter_rule); /* Inherit from the template. Don't inherit FILTRULES_SIDES; we check * that later. */ @@ -1125,8 +1117,7 @@ void parse_filter_str(filter_rule_list *listp, const char *rulestr, const char *name; filter_rule *excl_self; - if (!(excl_self = new0(filter_rule))) - out_of_memory("parse_filter_str"); + excl_self = new0(filter_rule); /* Find the beginning of the basename and add an exclude for it. */ for (name = pat + pat_len; name > pat && name[-1] != '/'; name--) {} add_rule(listp, name, (pat + pat_len) - name, excl_self, 0); diff --git a/fileio.c b/fileio.c index 32dc62da..f80af19e 100644 --- a/fileio.c +++ b/fileio.c @@ -157,8 +157,6 @@ int write_file(int f, int use_seek, OFF_T offset, const char *buf, int len) wf_writeBufSize = WRITE_SIZE * 8; wf_writeBufCnt = 0; wf_writeBuf = new_array(char, wf_writeBufSize); - if (!wf_writeBuf) - out_of_memory("write_file"); } r1 = (int)MIN((size_t)len, wf_writeBufSize - wf_writeBufCnt); if (r1) { @@ -217,8 +215,7 @@ struct map_struct *map_file(int fd, OFF_T len, int32 read_size, int32 blk_size) { struct map_struct *map; - if (!(map = new0(struct map_struct))) - out_of_memory("map_file"); + map = new0(struct map_struct); if (blk_size && (read_size % blk_size)) read_size += blk_size - (read_size % blk_size); @@ -261,8 +258,6 @@ char *map_ptr(struct map_struct *map, OFF_T offset, int32 len) /* make sure we have allocated enough memory for the window */ if (window_size > map->p_size) { map->p = realloc_array(map->p, char, window_size); - if (!map->p) - out_of_memory("map_ptr"); map->p_size = window_size; } diff --git a/flist.c b/flist.c index bbc028ba..f248e29b 100644 --- a/flist.c +++ b/flist.c @@ -301,8 +301,7 @@ static void flist_expand(struct file_list *flist, int extra) if (flist->malloced < flist->used + extra) flist->malloced = flist->used + extra; - new_ptr = realloc_array(flist->files, struct file_struct *, - flist->malloced); + new_ptr = realloc_array(flist->files, struct file_struct *, flist->malloced); if (DEBUG_GTE(FLIST, 1) && flist->malloced != FLIST_START) { rprintf(FCLIENT, "[%s] expand file_list pointer array to %s bytes, did%s move\n", @@ -312,9 +311,6 @@ static void flist_expand(struct file_list *flist, int extra) } flist->files = new_ptr; - - if (!flist->files) - out_of_memory("flist_expand"); } static void flist_done_allocating(struct file_list *flist) @@ -1335,10 +1331,8 @@ struct file_struct *make_file(const char *fname, struct file_list *flist, + linkname_len; if (pool) bp = pool_alloc(pool, alloc_len, "make_file"); - else { - if (!(bp = new_array(char, alloc_len))) - out_of_memory("make_file"); - } + else + bp = new_array(char, alloc_len); memset(bp, 0, extra_len + FILE_STRUCT_LEN); bp += extra_len; @@ -1661,8 +1655,7 @@ static void fsort(struct file_struct **fp, size_t num) if (use_qsort) qsort(fp, num, PTR_SIZE, file_compare); else { - struct file_struct **tmp = new_array(struct file_struct *, - (num+1) / 2); + struct file_struct **tmp = new_array(struct file_struct *, (num+1) / 2); fsort_tmp(fp, num, tmp); free(tmp); } @@ -1895,13 +1888,11 @@ static void send_implied_dirs(int f, struct file_list *flist, char *fname, len = strlen(limit+1); memcpy(&relname_list, F_DIR_RELNAMES_P(lastpath_struct), sizeof relname_list); if (!relname_list) { - if (!(relname_list = new0(item_list))) - out_of_memory("send_implied_dirs"); + relname_list = new0(item_list); memcpy(F_DIR_RELNAMES_P(lastpath_struct), &relname_list, sizeof relname_list); } rnpp = EXPAND_ITEM_LIST(relname_list, relnamecache *, 32); - if (!(*rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len))) - out_of_memory("send_implied_dirs"); + *rnpp = (relnamecache*)new_array(char, sizeof (relnamecache) + len); (*rnpp)->name_type = name_type; strlcpy((*rnpp)->fname, limit+1, len + 1); @@ -2059,8 +2050,7 @@ void send_extra_file_list(int f, int at_least) } if (need_unsorted_flist) { - if (!(flist->sorted = new_array(struct file_struct *, flist->used))) - out_of_memory("send_extra_file_list"); + flist->sorted = new_array(struct file_struct *, flist->used); memcpy(flist->sorted, flist->files, flist->used * sizeof (struct file_struct*)); } else @@ -2414,8 +2404,7 @@ struct file_list *send_file_list(int f, int argc, char *argv[]) * recursion mode, the sender marks duplicate dirs so that it can * send them together in a single file-list. */ if (need_unsorted_flist) { - if (!(flist->sorted = new_array(struct file_struct *, flist->used))) - out_of_memory("send_file_list"); + flist->sorted = new_array(struct file_struct *, flist->used); memcpy(flist->sorted, flist->files, flist->used * sizeof (struct file_struct*)); } else @@ -2597,8 +2586,7 @@ struct file_list *recv_file_list(int f, int dir_ndx) * order and for calling flist_find()). We keep the "files" * list unsorted for our exchange of index numbers with the * other side (since their names may not sort the same). */ - if (!(flist->sorted = new_array(struct file_struct *, flist->used))) - out_of_memory("recv_file_list"); + flist->sorted = new_array(struct file_struct *, flist->used); memcpy(flist->sorted, flist->files, flist->used * sizeof (struct file_struct*)); if (inc_recurse && dir_flist->used > dstart) { @@ -2808,8 +2796,7 @@ struct file_list *flist_new(int flags, char *msg) { struct file_list *flist; - if (!(flist = new0(struct file_list))) - out_of_memory(msg); + flist = new0(struct file_list); if (flags & FLIST_TEMP) { if (!(flist->file_pool = pool_create(SMALL_EXTENT, 0, diff --git a/generator.c b/generator.c index 8f9d6035..23a2eaff 100644 --- a/generator.c +++ b/generator.c @@ -2227,8 +2227,6 @@ void generate_files(int f_out, const char *local_name) if (delete_during == 2) { deldelay_size = BIGPATHBUFLEN * 4; deldelay_buf = new_array(char, deldelay_size); - if (!deldelay_buf) - out_of_memory("delete-delay"); } info_levels[INFO_FLIST] = info_levels[INFO_PROGRESS] = 0; diff --git a/getgroups.c b/getgroups.c index a96e04d4..8a37ed0b 100644 --- a/getgroups.c +++ b/getgroups.c @@ -3,7 +3,7 @@ * `id -G` on Linux, but it's too hard to find a portable equivalent. * * Copyright (C) 2002 Martin Pool - * Copyright (C) 2003-2019 Wayne Davison + * Copyright (C) 2003-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License version 3 as @@ -20,8 +20,7 @@ #include "rsync.h" -int -main(UNUSED(int argc), UNUSED(char *argv[])) + int main(UNUSED(int argc), UNUSED(char *argv[])) { int n, i; gid_t *list; diff --git a/hashtable.c b/hashtable.c index 17133dd2..e272f439 100644 --- a/hashtable.c +++ b/hashtable.c @@ -35,9 +35,8 @@ struct hashtable *hashtable_create(int size, int key64) size *= 2; } - if (!(tbl = new(struct hashtable)) - || !(tbl->nodes = new_array0(char, size * node_size))) - out_of_memory("hashtable_create"); + tbl = new(struct hashtable); + tbl->nodes = new_array0(char, size * node_size); tbl->size = size; tbl->entries = 0; tbl->node_size = node_size; @@ -94,8 +93,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new) int size = tbl->size * 2; int i; - if (!(tbl->nodes = new_array0(char, size * tbl->node_size))) - out_of_memory("hashtable_node"); + tbl->nodes = new_array0(char, size * tbl->node_size); tbl->size = size; tbl->entries = 0; diff --git a/hlink.c b/hlink.c index 85f54704..adec89b0 100644 --- a/hlink.c +++ b/hlink.c @@ -125,8 +125,7 @@ static void match_gnums(int32 *ndx_list, int ndx_count) if (inc_recurse) { node = hashtable_find(prior_hlinks, gnum, data_when_new); if (node->data == data_when_new) { - if (!(node->data = new_array0(char, 5))) - out_of_memory("match_gnums"); + node->data = new_array0(char, 5); assert(gnum >= hlink_flist->ndx_start); file->flags |= FLAG_HLINK_FIRST; prev = -1; @@ -190,8 +189,7 @@ void match_hard_links(struct file_list *flist) int i, ndx_count = 0; int32 *ndx_list; - if (!(ndx_list = new_array(int32, flist->used))) - out_of_memory("match_hard_links"); + ndx_list = new_array(int32, flist->used); for (i = 0; i < flist->used; i++) { if (F_IS_HLINKED(flist->sorted[i])) @@ -541,8 +539,7 @@ void finish_hard_link(struct file_struct *file, const char *fname, int fin_ndx, exit_cleanup(RERR_MESSAGEIO); } free(node->data); - if (!(node->data = strdup(our_name))) - out_of_memory("finish_hard_link"); + node->data = strdup(our_name); } } diff --git a/ifuncs.h b/ifuncs.h index 36ea51ad..b9490588 100644 --- a/ifuncs.h +++ b/ifuncs.h @@ -1,6 +1,6 @@ /* Inline functions for rsync. * - * Copyright (C) 2007-2019 Wayne Davison + * Copyright (C) 2007-2020 Wayne Davison * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,8 +19,7 @@ static inline void alloc_xbuf(xbuf *xb, size_t sz) { - if (!(xb->buf = new_array(char, sz))) - out_of_memory("alloc_xbuf"); + xb->buf = new_array(char, sz); xb->size = sz; xb->len = xb->pos = 0; } @@ -29,8 +28,6 @@ static inline void realloc_xbuf(xbuf *xb, size_t sz) { char *bf = realloc_array(xb->buf, char, sz); - if (!bf) - out_of_memory("realloc_xbuf"); xb->buf = bf; xb->size = sz; } @@ -104,3 +101,11 @@ free_stat_x(stat_x *sx_p) } #endif } + +static inline char *my_strdup(const char *str, const char *file, int line) +{ + int len = strlen(str)+1; + char *buf = my_alloc(do_malloc, len, 1, file, line); + memcpy(buf, str, len); + return buf; +} diff --git a/io.c b/io.c index 6c3730c8..ddd20fa8 100644 --- a/io.c +++ b/io.c @@ -1239,8 +1239,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, rl_flags |= (protect_args && ic_recv != (iconv_t)-1 ? RL_CONVERT : 0); #endif - if (!(argv = new_array(char *, maxargs))) - out_of_memory("read_args"); + argv = new_array(char *, maxargs); if (mod_name && !protect_args) argv[argc++] = "rsyncd"; @@ -1253,8 +1252,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, if (argc == maxargs-1) { maxargs += MAX_ARGS; - if (!(argv = realloc_array(argv, char *, maxargs))) - out_of_memory("read_args"); + argv = realloc_array(argv, char *, maxargs); } if (dot_pos) { @@ -1262,8 +1260,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, int len = strlen(buf); if (request_len) request_p[0][request_len++] = ' '; - if (!(*request_p = realloc_array(*request_p, char, request_len + len + 1))) - out_of_memory("read_args"); + *request_p = realloc_array(*request_p, char, request_len + len + 1); memcpy(*request_p + request_len, buf, len + 1); request_len += len; } @@ -1272,8 +1269,7 @@ void read_args(int f_in, char *mod_name, char *buf, size_t bufsiz, int rl_nulls, else glob_expand(buf, &argv, &argc, &maxargs); } else { - if (!(p = strdup(buf))) - out_of_memory("read_args"); + p = strdup(buf); argv[argc++] = p; if (*p == '.' && p[1] == '\0') dot_pos = argc; diff --git a/lib/md-defines.h b/lib/md-defines.h index b92e8c07..1410af5f 100644 --- a/lib/md-defines.h +++ b/lib/md-defines.h @@ -13,3 +13,5 @@ #define CSUM_MD4 4 #define CSUM_MD5 5 #define CSUM_XXH64 6 +#define CSUM_XXH3_64 7 +#define CSUM_XXH3_128 8 diff --git a/lib/md5-asm-x86_64.S b/lib/md5-asm-x86_64.S index 814aac59..383f193a 100644 --- a/lib/md5-asm-x86_64.S +++ b/lib/md5-asm-x86_64.S @@ -29,6 +29,10 @@ #if !defined USE_OPENSSL && CSUM_CHUNK == 64 +#ifdef __APPLE__ +#define md5_process_asm _md5_process_asm +#endif + .text .align 16 diff --git a/loadparm.c b/loadparm.c index dfc0b077..a1d5992d 100644 --- a/loadparm.c +++ b/loadparm.c @@ -42,12 +42,12 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" #include "default-dont-compress.h" extern item_list dparam_list; #define strequal(a, b) (strcasecmp(a, b)==0) -#define BOOLSTR(b) ((b) ? "Yes" : "No") #ifndef LOG_DAEMON #define LOG_DAEMON 0 @@ -55,7 +55,7 @@ extern item_list dparam_list; /* the following are used by loadparm for option lists */ typedef enum { - P_BOOL, P_BOOLREV, P_CHAR, P_INTEGER, + P_BOOL, P_BOOLREV, P_BOOL3, P_CHAR, P_INTEGER, P_OCTAL, P_PATH, P_STRING, P_ENUM } parm_type; @@ -86,234 +86,6 @@ struct parm_struct { #define LP_SNUM_OK(i) ((i) >= 0 && (i) < (int)section_list.count) #define SECTION_PTR(s, p) (((char*)(s)) + (ptrdiff_t)(((char*)(p))-(char*)&Vars.l)) -/* This structure describes global (ie., server-wide) parameters. */ -typedef struct { - char *bind_address; - char *daemon_chroot; - char *daemon_gid; - char *daemon_uid; - char *motd_file; - char *pid_file; - char *socket_options; - - /* Each _EXP var tracks if the associated char* var has been expanded yet or not. */ - BOOL bind_address_EXP; - BOOL daemon_chroot_EXP; - BOOL daemon_gid_EXP; - BOOL daemon_uid_EXP; - BOOL motd_file_EXP; - BOOL pid_file_EXP; - BOOL socket_options_EXP; - - int listen_backlog; - int rsync_port; - - BOOL proxy_protocol; -} global_vars; - -/* This structure describes a single section. Their order must match the - * initializers below, which you can accomplish by keeping each sub-section - * sorted. (e.g. in vim, just visually select each subsection and use !sort.) - * NOTE: the char* variables MUST all remain at the start of the struct! */ -typedef struct { - char *auth_users; - char *charset; - char *comment; - char *dont_compress; - char *early_exec; - char *exclude; - char *exclude_from; - char *filter; - char *gid; - char *hosts_allow; - char *hosts_deny; - char *include; - char *include_from; - char *incoming_chmod; - char *lock_file; - char *log_file; - char *log_format; - char *name; - char *outgoing_chmod; - char *path; - char *postxfer_exec; - char *prexfer_exec; - char *refuse_options; - char *secrets_file; - char *syslog_tag; - char *temp_dir; - char *uid; - - /* Each _EXP var tracks if the associated char* var has been expanded yet or not. */ - BOOL auth_users_EXP; - BOOL charset_EXP; - BOOL comment_EXP; - BOOL dont_compress_EXP; - BOOL early_exec_EXP; - BOOL exclude_EXP; - BOOL exclude_from_EXP; - BOOL filter_EXP; - BOOL gid_EXP; - BOOL hosts_allow_EXP; - BOOL hosts_deny_EXP; - BOOL include_EXP; - BOOL include_from_EXP; - BOOL incoming_chmod_EXP; - BOOL lock_file_EXP; - BOOL log_file_EXP; - BOOL log_format_EXP; - BOOL name_EXP; - BOOL outgoing_chmod_EXP; - BOOL path_EXP; - BOOL postxfer_exec_EXP; - BOOL prexfer_exec_EXP; - BOOL refuse_options_EXP; - BOOL secrets_file_EXP; - BOOL syslog_tag_EXP; - BOOL temp_dir_EXP; - BOOL uid_EXP; - - int max_connections; - int max_verbosity; - int syslog_facility; - int timeout; - - BOOL fake_super; - BOOL forward_lookup; - BOOL ignore_errors; - BOOL ignore_nonreadable; - BOOL list; - BOOL munge_symlinks; - BOOL numeric_ids; - BOOL read_only; - BOOL reverse_lookup; - BOOL strict_modes; - BOOL transfer_logging; - BOOL use_chroot; - BOOL write_only; -} local_vars; - -/* This structure describes the global variables (g) as well as the globally - * specified values of the local variables (l), which are used when modules - * don't specify their own values. */ -typedef struct { - global_vars g; - local_vars l; -} all_vars; - -/* The application defaults for all the variables. "Defaults" is - * used to re-initialize "Vars" before each config-file read. - * - * In order to keep these sorted in the same way as the structure - * above, use the variable name in the leading comment, including a - * trailing ';' (to avoid a sorting problem with trailing digits). */ -static const all_vars Defaults = { - /* ==== global_vars ==== */ - { - /* bind_address; */ NULL, - /* daemon_chroot; */ NULL, - /* daemon_gid; */ NULL, - /* daemon_uid; */ NULL, - /* motd_file; */ NULL, - /* pid_file; */ NULL, - /* socket_options; */ NULL, - - /* bind_address_EXP; */ False, - /* daemon_chroot_EXP; */ False, - /* daemon_gid_EXP; */ False, - /* daemon_uid_EXP; */ False, - /* motd_file_EXP; */ False, - /* pid_file_EXP; */ False, - /* socket_options_EXP; */ False, - - /* listen_backlog; */ 5, - /* rsync_port; */ 0, - - /* proxy_protocol; */ False, - }, - - /* ==== local_vars ==== */ - { - /* auth_users; */ NULL, - /* charset; */ NULL, - /* comment; */ NULL, - /* dont_compress; */ DEFAULT_DONT_COMPRESS, - /* early_exec; */ NULL, - /* exclude; */ NULL, - /* exclude_from; */ NULL, - /* filter; */ NULL, - /* gid; */ NULL, - /* hosts_allow; */ NULL, - /* hosts_deny; */ NULL, - /* include; */ NULL, - /* include_from; */ NULL, - /* incoming_chmod; */ NULL, - /* lock_file; */ DEFAULT_LOCK_FILE, - /* log_file; */ NULL, - /* log_format; */ "%o %h [%a] %m (%u) %f %l", - /* name; */ NULL, - /* outgoing_chmod; */ NULL, - /* path; */ NULL, - /* postxfer_exec; */ NULL, - /* prexfer_exec; */ NULL, - /* refuse_options; */ NULL, - /* secrets_file; */ NULL, - /* syslog_tag; */ "rsyncd", - /* temp_dir; */ NULL, - /* uid; */ NULL, - - /* auth_users_EXP; */ False, - /* charset_EXP; */ False, - /* comment_EXP; */ False, - /* dont_compress_EXP; */ False, - /* early_exec_EXP; */ False, - /* exclude_EXP; */ False, - /* exclude_from_EXP; */ False, - /* filter_EXP; */ False, - /* gid_EXP; */ False, - /* hosts_allow_EXP; */ False, - /* hosts_deny_EXP; */ False, - /* include_EXP; */ False, - /* include_from_EXP; */ False, - /* incoming_chmod_EXP; */ False, - /* lock_file_EXP; */ False, - /* log_file_EXP; */ False, - /* log_format_EXP; */ False, - /* name_EXP; */ False, - /* outgoing_chmod_EXP; */ False, - /* path_EXP; */ False, - /* postxfer_exec_EXP; */ False, - /* prexfer_exec_EXP; */ False, - /* refuse_options_EXP; */ False, - /* secrets_file_EXP; */ False, - /* syslog_tag_EXP; */ False, - /* temp_dir_EXP; */ False, - /* uid_EXP; */ False, - - /* max_connections; */ 0, - /* max_verbosity; */ 1, - /* syslog_facility; */ LOG_DAEMON, - /* timeout; */ 0, - - /* fake_super; */ False, - /* forward_lookup; */ True, - /* ignore_errors; */ False, - /* ignore_nonreadable; */ False, - /* list; */ True, - /* munge_symlinks; */ (BOOL)-1, - /* numeric_ids; */ (BOOL)-1, - /* read_only; */ True, - /* reverse_lookup; */ True, - /* strict_modes; */ True, - /* transfer_logging; */ False, - /* use_chroot; */ True, - /* write_only; */ False, - } -}; - -/* The currently configured values for all the variables. */ -static all_vars Vars; - /* Stack of "Vars" values used by the &include directive. */ static item_list Vars_stack = EMPTY_ITEM_LIST; @@ -323,9 +95,7 @@ static item_list section_list = EMPTY_ITEM_LIST; static int iSectionIndex = -1; static BOOL bInGlobalSection = True; -#define NUMPARAMETERS (sizeof (parm_table) / sizeof (struct parm_struct)) - -static struct enum_list enum_facilities[] = { +static struct enum_list enum_syslog_facility[] = { #ifdef LOG_AUTH { LOG_AUTH, "auth" }, #endif @@ -392,96 +162,27 @@ static struct enum_list enum_facilities[] = { { -1, NULL } }; -static struct parm_struct parm_table[] = -{ - {"address", P_STRING, P_GLOBAL,&Vars.g.bind_address, NULL,0}, - {"daemon chroot", P_STRING, P_GLOBAL,&Vars.g.daemon_chroot, NULL,0}, - {"daemon gid", P_STRING, P_GLOBAL,&Vars.g.daemon_gid, NULL,0}, - {"daemon uid", P_STRING, P_GLOBAL,&Vars.g.daemon_uid, NULL,0}, - {"listen backlog", P_INTEGER,P_GLOBAL,&Vars.g.listen_backlog, NULL,0}, - {"motd file", P_STRING, P_GLOBAL,&Vars.g.motd_file, NULL,0}, - {"pid file", P_STRING, P_GLOBAL,&Vars.g.pid_file, NULL,0}, - {"port", P_INTEGER,P_GLOBAL,&Vars.g.rsync_port, NULL,0}, - {"proxy protocol", P_BOOL, P_LOCAL, &Vars.g.proxy_protocol, NULL,0}, - {"socket options", P_STRING, P_GLOBAL,&Vars.g.socket_options, NULL,0}, - - {"auth users", P_STRING, P_LOCAL, &Vars.l.auth_users, NULL,0}, - {"charset", P_STRING, P_LOCAL, &Vars.l.charset, NULL,0}, - {"comment", P_STRING, P_LOCAL, &Vars.l.comment, NULL,0}, - {"dont compress", P_STRING, P_LOCAL, &Vars.l.dont_compress, NULL,0}, - {"early exec", P_STRING, P_LOCAL, &Vars.l.early_exec, NULL,0}, - {"exclude from", P_STRING, P_LOCAL, &Vars.l.exclude_from, NULL,0}, - {"exclude", P_STRING, P_LOCAL, &Vars.l.exclude, NULL,0}, - {"fake super", P_BOOL, P_LOCAL, &Vars.l.fake_super, NULL,0}, - {"filter", P_STRING, P_LOCAL, &Vars.l.filter, NULL,0}, - {"forward lookup", P_BOOL, P_LOCAL, &Vars.l.forward_lookup, NULL,0}, - {"gid", P_STRING, P_LOCAL, &Vars.l.gid, NULL,0}, - {"hosts allow", P_STRING, P_LOCAL, &Vars.l.hosts_allow, NULL,0}, - {"hosts deny", P_STRING, P_LOCAL, &Vars.l.hosts_deny, NULL,0}, - {"ignore errors", P_BOOL, P_LOCAL, &Vars.l.ignore_errors, NULL,0}, - {"ignore nonreadable",P_BOOL, P_LOCAL, &Vars.l.ignore_nonreadable, NULL,0}, - {"include from", P_STRING, P_LOCAL, &Vars.l.include_from, NULL,0}, - {"include", P_STRING, P_LOCAL, &Vars.l.include, NULL,0}, - {"incoming chmod", P_STRING, P_LOCAL, &Vars.l.incoming_chmod, NULL,0}, - {"list", P_BOOL, P_LOCAL, &Vars.l.list, NULL,0}, - {"lock file", P_STRING, P_LOCAL, &Vars.l.lock_file, NULL,0}, - {"log file", P_STRING, P_LOCAL, &Vars.l.log_file, NULL,0}, - {"log format", P_STRING, P_LOCAL, &Vars.l.log_format, NULL,0}, - {"max connections", P_INTEGER,P_LOCAL, &Vars.l.max_connections, NULL,0}, - {"max verbosity", P_INTEGER,P_LOCAL, &Vars.l.max_verbosity, NULL,0}, - {"munge symlinks", P_BOOL, P_LOCAL, &Vars.l.munge_symlinks, NULL,0}, - {"name", P_STRING, P_LOCAL, &Vars.l.name, NULL,0}, - {"numeric ids", P_BOOL, P_LOCAL, &Vars.l.numeric_ids, NULL,0}, - {"outgoing chmod", P_STRING, P_LOCAL, &Vars.l.outgoing_chmod, NULL,0}, - {"path", P_PATH, P_LOCAL, &Vars.l.path, NULL,0}, -#ifdef HAVE_PUTENV - {"post-xfer exec", P_STRING, P_LOCAL, &Vars.l.postxfer_exec, NULL,0}, - {"pre-xfer exec", P_STRING, P_LOCAL, &Vars.l.prexfer_exec, NULL,0}, -#endif - {"read only", P_BOOL, P_LOCAL, &Vars.l.read_only, NULL,0}, - {"refuse options", P_STRING, P_LOCAL, &Vars.l.refuse_options, NULL,0}, - {"reverse lookup", P_BOOL, P_LOCAL, &Vars.l.reverse_lookup, NULL,0}, - {"secrets file", P_STRING, P_LOCAL, &Vars.l.secrets_file, NULL,0}, - {"strict modes", P_BOOL, P_LOCAL, &Vars.l.strict_modes, NULL,0}, - {"syslog facility", P_ENUM, P_LOCAL, &Vars.l.syslog_facility, enum_facilities,0}, - {"syslog tag", P_STRING, P_LOCAL, &Vars.l.syslog_tag, NULL,0}, - {"temp dir", P_PATH, P_LOCAL, &Vars.l.temp_dir, NULL,0}, - {"timeout", P_INTEGER,P_LOCAL, &Vars.l.timeout, NULL,0}, - {"transfer logging", P_BOOL, P_LOCAL, &Vars.l.transfer_logging, NULL,0}, - {"uid", P_STRING, P_LOCAL, &Vars.l.uid, NULL,0}, - {"use chroot", P_BOOL, P_LOCAL, &Vars.l.use_chroot, NULL,0}, - {"write only", P_BOOL, P_LOCAL, &Vars.l.write_only, NULL,0}, - {NULL, P_BOOL, P_NONE, NULL, NULL,0} -}; - -/* Initialise the Default all_vars structure. */ -void reset_daemon_vars(void) -{ - memcpy(&Vars, &Defaults, sizeof Vars); -} - /* Expand %VAR% references. Any unknown vars or unrecognized * syntax leaves the raw chars unchanged. */ -static char *expand_vars(char *str) +static char *expand_vars(const char *str) { - char *buf, *t, *f; + char *buf, *t; + const char *f; int bufsize; if (!str || !strchr(str, '%')) - return str; + return (char *)str; /* TODO change return value to const char* at some point. */ bufsize = strlen(str) + 2048; - if ((buf = new_array(char, bufsize+1)) == NULL) /* +1 for trailing '\0' */ - out_of_memory("expand_vars"); + buf = new_array(char, bufsize+1); /* +1 for trailing '\0' */ for (t = buf, f = str; bufsize && *f; ) { - if (*f == '%' && *++f != '%') { - char *percent = strchr(f, '%'); - if (percent) { + if (*f == '%' && isUpper(f+1)) { + char *percent = strchr(f+1, '%'); + if (percent && percent - f < bufsize) { char *val; - *percent = '\0'; - val = getenv(f); - *percent = '%'; + strlcpy(t, f+1, percent - f); + val = getenv(t); if (val) { int len = strlcpy(t, val, bufsize+1); if (len > bufsize) @@ -492,7 +193,6 @@ static char *expand_vars(char *str) continue; } } - f--; } *t++ = *f++; bufsize--; @@ -510,6 +210,8 @@ static char *expand_vars(char *str) return buf; } +/* Each "char* foo" has an associated "BOOL foo_EXP" that tracks if the string has been expanded yet or not. */ + /* NOTE: use this function and all the FN_{GLOBAL,LOCAL} ones WITHOUT a trailing semicolon! */ #define RETURN_EXPANDED(val) {if (!val ## _EXP) {val = expand_vars(val); val ## _EXP = True;} return val ? val : "";} @@ -534,65 +236,24 @@ static char *expand_vars(char *str) #define FN_LOCAL_INTEGER(fn_name, val) \ int fn_name(int i) {return LP_SNUM_OK(i)? iSECTION(i).val : Vars.l.val;} -FN_GLOBAL_STRING(lp_bind_address, bind_address) -FN_GLOBAL_STRING(lp_daemon_chroot, daemon_chroot) -FN_GLOBAL_STRING(lp_daemon_gid, daemon_gid) -FN_GLOBAL_STRING(lp_daemon_uid, daemon_uid) -FN_GLOBAL_STRING(lp_motd_file, motd_file) -FN_GLOBAL_STRING(lp_pid_file, pid_file) -FN_GLOBAL_STRING(lp_socket_options, socket_options) +/* The following include file contains: + * + * typedef global_vars - describes global (ie., server-wide) parameters. + * typedef local_vars - describes a single section. + * typedef all_vars - a combination of global_vars & local_vars. + * all_vars Defaults - the default values for all the variables. + * all_vars Vars - tThe currently configured values for all the variables. + * struct parm_struct parm_table - the strings & variables for the parser. + * FN_{LOCAL,GLOBAL}_{TYPE}() definition for all the lp_var_name() accessors. + */ -FN_GLOBAL_INTEGER(lp_listen_backlog, listen_backlog) -FN_GLOBAL_INTEGER(lp_rsync_port, rsync_port) +#include "daemon-parm.h" -FN_GLOBAL_BOOL(lp_proxy_protocol, proxy_protocol) - -FN_LOCAL_STRING(lp_auth_users, auth_users) -FN_LOCAL_STRING(lp_charset, charset) -FN_LOCAL_STRING(lp_comment, comment) -FN_LOCAL_STRING(lp_dont_compress, dont_compress) -FN_LOCAL_STRING(lp_early_exec, early_exec) -FN_LOCAL_STRING(lp_exclude, exclude) -FN_LOCAL_STRING(lp_exclude_from, exclude_from) -FN_LOCAL_STRING(lp_filter, filter) -FN_LOCAL_STRING(lp_gid, gid) -FN_LOCAL_STRING(lp_hosts_allow, hosts_allow) -FN_LOCAL_STRING(lp_hosts_deny, hosts_deny) -FN_LOCAL_STRING(lp_include, include) -FN_LOCAL_STRING(lp_include_from, include_from) -FN_LOCAL_STRING(lp_incoming_chmod, incoming_chmod) -FN_LOCAL_STRING(lp_lock_file, lock_file) -FN_LOCAL_STRING(lp_log_file, log_file) -FN_LOCAL_STRING(lp_log_format, log_format) -FN_LOCAL_STRING(lp_name, name) -FN_LOCAL_STRING(lp_outgoing_chmod, outgoing_chmod) -FN_LOCAL_STRING(lp_path, path) -FN_LOCAL_STRING(lp_postxfer_exec, postxfer_exec) -FN_LOCAL_STRING(lp_prexfer_exec, prexfer_exec) -FN_LOCAL_STRING(lp_refuse_options, refuse_options) -FN_LOCAL_STRING(lp_secrets_file, secrets_file) -FN_LOCAL_STRING(lp_syslog_tag, syslog_tag) -FN_LOCAL_STRING(lp_temp_dir, temp_dir) -FN_LOCAL_STRING(lp_uid, uid) - -FN_LOCAL_INTEGER(lp_max_connections, max_connections) -FN_LOCAL_INTEGER(lp_max_verbosity, max_verbosity) -FN_LOCAL_INTEGER(lp_syslog_facility, syslog_facility) -FN_LOCAL_INTEGER(lp_timeout, timeout) - -FN_LOCAL_BOOL(lp_fake_super, fake_super) -FN_LOCAL_BOOL(lp_forward_lookup, forward_lookup) -FN_LOCAL_BOOL(lp_ignore_errors, ignore_errors) -FN_LOCAL_BOOL(lp_ignore_nonreadable, ignore_nonreadable) -FN_LOCAL_BOOL(lp_list, list) -FN_LOCAL_BOOL(lp_munge_symlinks, munge_symlinks) -FN_LOCAL_BOOL(lp_numeric_ids, numeric_ids) -FN_LOCAL_BOOL(lp_read_only, read_only) -FN_LOCAL_BOOL(lp_reverse_lookup, reverse_lookup) -FN_LOCAL_BOOL(lp_strict_modes, strict_modes) -FN_LOCAL_BOOL(lp_transfer_logging, transfer_logging) -FN_LOCAL_BOOL(lp_use_chroot, use_chroot) -FN_LOCAL_BOOL(lp_write_only, write_only) +/* Initialise the Default all_vars structure. */ +void reset_daemon_vars(void) +{ + memcpy(&Vars, &Defaults, sizeof Vars); +} /* Assign a copy of v to *s. Handles NULL strings. We don't worry * about overwriting a malloc'd string because the long-running @@ -601,10 +262,7 @@ FN_LOCAL_BOOL(lp_write_only, write_only) * the start, so any lost memory is inconsequential. */ static inline void string_set(char **s, const char *v) { - if (!v) - *s = NULL; - else if (!(*s = strdup(v))) - out_of_memory("string_set"); + *s = v ? strdup(v) : NULL; } /* Copy local_vars into a new section. No need to strdup since we don't free. */ @@ -620,19 +278,14 @@ static void init_section(local_vars *psection) copy_section(psection, &Vars.l); } -/* Do a case-insensitive, whitespace-ignoring string compare. */ -static int strwicmp(char *psz1, char *psz2) +/* Do a case-insensitive, whitespace-ignoring string equality check. */ +static int strwiEQ(char *psz1, char *psz2) { - /* if BOTH strings are NULL, return TRUE, if ONE is NULL return */ - /* appropriate value. */ + /* If one or both strings are NULL, we return equality right away. */ if (psz1 == psz2) - return 0; - - if (psz1 == NULL) - return -1; - - if (psz2 == NULL) return 1; + if (psz1 == NULL || psz2 == NULL) + return 0; /* sync the strings on first non-whitespace */ while (1) { @@ -640,12 +293,14 @@ static int strwicmp(char *psz1, char *psz2) psz1++; while (isSpace(psz2)) psz2++; - if (toUpper(psz1) != toUpper(psz2) || *psz1 == '\0' || *psz2 == '\0') + if (*psz1 == '\0' || *psz2 == '\0') + break; + if (toUpper(psz1) != toUpper(psz2)) break; psz1++; psz2++; } - return *psz1 - *psz2; + return *psz1 == *psz2; } /* Find a section by name. Otherwise works like get_section. */ @@ -654,7 +309,7 @@ static int getsectionbyname(char *name) int i; for (i = section_list.count - 1; i >= 0; i--) { - if (strwicmp(iSECTION(i).name, name) == 0) + if (strwiEQ(iSECTION(i).name, name)) break; } @@ -694,7 +349,7 @@ static int map_parameter(char *parmname) return -1; for (iIndex = 0; parm_table[iIndex].label; iIndex++) { - if (strwicmp(parm_table[iIndex].label, parmname) == 0) + if (strwiEQ(parm_table[iIndex].label, parmname)) return iIndex; } @@ -705,16 +360,14 @@ static int map_parameter(char *parmname) /* Set a boolean variable from the text value stored in the passed string. * Returns True in success, False if the passed string does not correctly * represent a boolean. */ -static BOOL set_boolean(BOOL *pb, char *parmvalue) +static BOOL set_boolean(BOOL *pb, char *parmvalue, int allow_unset) { - if (strwicmp(parmvalue, "yes") == 0 - || strwicmp(parmvalue, "true") == 0 - || strwicmp(parmvalue, "1") == 0) + if (strwiEQ(parmvalue, "yes") || strwiEQ(parmvalue, "true") || strwiEQ(parmvalue, "1")) *pb = True; - else if (strwicmp(parmvalue, "no") == 0 - || strwicmp(parmvalue, "False") == 0 - || strwicmp(parmvalue, "0") == 0) + else if (strwiEQ(parmvalue, "no") || strwiEQ(parmvalue, "false") || strwiEQ(parmvalue, "0")) *pb = False; + else if (allow_unset && (strwiEQ(parmvalue, "unset") || strwiEQ(parmvalue, "-1"))) + *pb = Unset; else { rprintf(FLOG, "Badly formed boolean in configuration file: \"%s\".\n", parmvalue); return False; @@ -763,11 +416,15 @@ static BOOL do_parameter(char *parmname, char *parmvalue) switch (parm_table[parmnum].type) { case P_BOOL: - set_boolean(parm_ptr, parmvalue); + set_boolean(parm_ptr, parmvalue, False); + break; + + case P_BOOL3: + set_boolean(parm_ptr, parmvalue, True); break; case P_BOOLREV: - set_boolean(parm_ptr, parmvalue); + set_boolean(parm_ptr, parmvalue, False); *(BOOL *)parm_ptr = ! *(BOOL *)parm_ptr; break; @@ -837,7 +494,7 @@ static BOOL do_section(char *sectionname) return True; } - isglobal = strwicmp(sectionname, GLOBAL_NAME) == 0; + isglobal = strwiEQ(sectionname, GLOBAL_NAME); /* At the end of the global section, add any --dparam items. */ if (bInGlobalSection && !isglobal) { diff --git a/main.c b/main.c index b41a3943..c832d575 100644 --- a/main.c +++ b/main.c @@ -22,6 +22,7 @@ #include "rsync.h" #include "inums.h" +#include "ifuncs.h" #include "io.h" #if defined CONFIG_LOCALE && defined HAVE_LOCALE_H #include @@ -512,8 +513,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in if (!cmd) cmd = RSYNC_RSH; cmd = need_to_free = strdup(cmd); - if (!cmd) - goto oom; for (t = f = cmd; *f; f++) { if (*f == ' ') @@ -657,10 +656,6 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in free(need_to_free); return pid; - - oom: - out_of_memory("do_cmd"); - return 0; /* not reached */ } /* The receiving side operates in one of two modes: @@ -824,8 +819,6 @@ static void check_alt_basis_dirs(void) if (dry_run > 1 && *bdir != '/') { int len = curr_dir_len + 1 + bd_len + 1; char *new = new_array(char, len); - if (!new) - out_of_memory("check_alt_basis_dirs"); if (slash && strncmp(bdir, "../", 3) == 0) { /* We want to remove only one leading "../" prefix for * the directory we couldn't create in dry-run mode: @@ -1339,19 +1332,12 @@ int client_run(int f_in, int f_out, pid_t pid, int argc, char *argv[]) return MAX(exit_code, exit_code2); } -static int copy_argv(char *argv[]) +static void dup_argv(char *argv[]) { int i; - for (i = 0; argv[i]; i++) { - if (!(argv[i] = strdup(argv[i]))) { - rprintf (FERROR, "out of memory at %s(%d)\n", - __FILE__, __LINE__); - return RERR_MALLOC; - } - } - - return 0; + for (i = 0; argv[i]; i++) + argv[i] = strdup(argv[i]); } @@ -1372,8 +1358,7 @@ static int start_client(int argc, char *argv[]) /* Don't clobber argv[] so that ps(1) can still show the right * command line. */ - if ((ret = copy_argv(argv)) != 0) - return ret; + dup_argv(argv); if (!read_batch) { /* for read_batch, NO source is specified */ char *path = check_for_hostspec(argv[0], &shell_machine, &rsync_port); diff --git a/match.c b/match.c index 4ae7aa46..9d5c9259 100644 --- a/match.c +++ b/match.c @@ -65,8 +65,6 @@ static void build_hash_table(struct sum_struct *s) if (hash_table) free(hash_table); hash_table = new_array(int32, tablesize); - if (!hash_table) - out_of_memory("build_hash_table"); alloc_size = tablesize; } diff --git a/md2man b/md2man index ef4e58d6..992d7320 100755 --- a/md2man +++ b/md2man @@ -93,7 +93,7 @@ def main(): env_subs['VERSION'] = '1.0.0' env_subs['libdir'] = '/usr' else: - for fn in (fi.srcdir + 'NEWS.md', 'Makefile'): + for fn in (fi.srcdir + 'version.h', 'Makefile'): try: st = os.lstat(fn) except: @@ -101,6 +101,11 @@ def main(): if not fi.mtime: fi.mtime = st.st_mtime + with open(fi.srcdir + 'version.h', 'r', encoding='utf-8') as fh: + txt = fh.read() + m = re.search(r'"(.+?)"', txt) + env_subs['VERSION'] = m.group(1) + with open('Makefile', 'r', encoding='utf-8') as fh: for line in fh: m = re.match(r'^(\w+)=(.+)', line) @@ -112,7 +117,7 @@ def main(): while re.search(r'\$\{', val): val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val) env_subs[var] = val - if var == 'VERSION': + if var == 'srcdir': break with open(fi.fn, 'r', encoding='utf-8') as fh: diff --git a/options.c b/options.c index a5038703..d498e226 100644 --- a/options.c +++ b/options.c @@ -21,6 +21,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" #include "latest-year.h" #include @@ -181,6 +182,10 @@ int rsync_port = 0; int alt_dest_type = 0; int basis_dir_cnt = 0; +#define DEFAULT_MAX_ALLOC (1024L * 1024 * 1024) +size_t max_alloc = DEFAULT_MAX_ALLOC; +char *max_alloc_arg; + static int version_opt_cnt = 0; static int remote_option_alloc = 0; int remote_option_cnt = 0; @@ -382,8 +387,7 @@ static char *make_output_option(struct output_struct *words, short *levels, ucha return NULL; len++; - if (!(buf = new_array(char, len))) - out_of_memory("make_output_option"); + buf = new_array(char, len); pos = 0; if (skipped || max < 5) @@ -577,12 +581,15 @@ static char *istring(const char *fmt, int val) return str; } -static void print_capabilities(enum logcode f) +static void print_info_flags(enum logcode f) { STRUCT_STAT *dumstat; char line_buf[75]; int line_len, j; char *capabilities[] = { + + "*Capabilities", + istring("%d-bit files", (int)(sizeof (OFF_T) * 8)), istring("%d-bit inums", (int)(sizeof dumstat->st_ino * 8)), /* Don't check ino_t! */ istring("%d-bit timestamps", (int)(sizeof (time_t) * 8)), @@ -657,7 +664,8 @@ static void print_capabilities(enum logcode f) #endif "prealloc", - "*" /* All options after this point are hidden w/o -V -V */ + "*Optimizations", + #ifndef HAVE_SIMD "no " #endif @@ -677,28 +685,19 @@ static void print_capabilities(enum logcode f) }; for (line_len = 0, j = 0; ; j++) { - char *cap = capabilities[j]; - if (!cap) - break; - if (*cap == '*') { - if (version_opt_cnt >= 2) - capabilities[j]++; - else - capabilities[j] = NULL; - break; - } - } - - for (line_len = 0, j = 0; ; j++) { - char *cap = capabilities[j]; - int cap_len = cap ? strlen(cap) : 1000; - int need_comma = cap && capabilities[j+1] != NULL ? 1 : 0; - if (line_len + 1 + cap_len + need_comma >= (int)sizeof line_buf) { + char *cap = capabilities[j], *next_cap = cap ? capabilities[j+1] : NULL; + int cap_len = cap && *cap != '*' ? strlen(cap) : 1000; + int need_comma = next_cap && *next_cap != '*' ? 1 : 0; + if (line_len && line_len + 1 + cap_len + need_comma >= (int)sizeof line_buf) { rprintf(f, " %s\n", line_buf); line_len = 0; } if (!cap) break; + if (*cap == '*') { + rprintf(f, "%s:\n", cap+1); + continue; + } line_len += snprintf(line_buf+line_len, sizeof line_buf - line_len, " %s%s", cap, need_comma ? "," : ""); } } @@ -716,8 +715,7 @@ static void print_rsync_version(enum logcode f) rprintf(f, "Copyright (C) 1996-" LATEST_YEAR " by Andrew Tridgell, Wayne Davison, and others.\n"); rprintf(f, "Web site: https://rsync.samba.org/\n"); - rprintf(f, "Capabilities:\n"); - print_capabilities(f); + print_info_flags(f); rprintf(f, "Checksum list:\n"); get_default_nno_list(&valid_checksums, tmpbuf, sizeof tmpbuf, '('); @@ -889,6 +887,7 @@ static struct poptOption long_options[] = { {"ignore-existing", 0, POPT_ARG_NONE, &ignore_existing, 0, 0, 0 }, {"max-size", 0, POPT_ARG_STRING, &max_size_arg, OPT_MAX_SIZE, 0, 0 }, {"min-size", 0, POPT_ARG_STRING, &min_size_arg, OPT_MIN_SIZE, 0, 0 }, + {"max-alloc", 0, POPT_ARG_STRING, &max_alloc_arg, 0, 0, 0 }, {"sparse", 'S', POPT_ARG_VAL, &sparse_files, 1, 0, 0 }, {"no-sparse", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, {"no-S", 0, POPT_ARG_VAL, &sparse_files, 0, 0, 0 }, @@ -1251,14 +1250,16 @@ static int count_args(const char **argv) return i; } - -static OFF_T parse_size_arg(char **size_arg, char def_suf) +/* If the size_arg is an invalid string or the value is < min_value, an error + * is put into err_buf & the return is -1. Note that this parser does NOT + * support negative numbers, so a min_value < 0 doesn't make any sense. */ +static ssize_t parse_size_arg(char *size_arg, char def_suf, const char *opt_name, ssize_t min_value) { - int reps, mult, make_compatible = 0; - const char *arg; - OFF_T size = 1; + int reps, mult; + const char *arg, *err = "invalid"; + ssize_t size = 1; - for (arg = *size_arg; isDigit(arg); arg++) {} + for (arg = size_arg; isDigit(arg); arg++) {} if (*arg == '.') for (arg++; isDigit(arg); arg++) {} switch (*arg && *arg != '+' && *arg != '-' ? *arg++ : def_suf) { @@ -1274,40 +1275,40 @@ static OFF_T parse_size_arg(char **size_arg, char def_suf) case 'g': case 'G': reps = 3; break; + case 't': case 'T': + reps = 4; + break; + case 'p': case 'P': + reps = 5; + break; default: - return -1; + goto failure; } if (*arg == 'b' || *arg == 'B') - mult = 1000, make_compatible = 1, arg++; + mult = 1000, arg++; else if (!*arg || *arg == '+' || *arg == '-') mult = 1024; else if (strncasecmp(arg, "ib", 2) == 0) mult = 1024, arg += 2; else - return -1; + goto failure; while (reps--) size *= mult; - size *= atof(*size_arg); - if ((*arg == '+' || *arg == '-') && arg[1] == '1') - size += atoi(arg), make_compatible = 1, arg += 2; + size *= atof(size_arg); + if ((*arg == '+' || *arg == '-') && arg[1] == '1' && arg != size_arg) + size += atoi(arg), arg += 2; if (*arg) - return -1; - if (size > 0 && make_compatible && def_suf == 'b') { - /* We convert this manually because we may need %lld precision, - * and that's not a portable sprintf() escape. */ - char buf[128], *s = buf + sizeof buf - 1; - OFF_T num = size; - *s = '\0'; - while (num) { - *--s = (char)(num % 10) + '0'; - num /= 10; - } - if (!(*size_arg = strdup(s))) - out_of_memory("parse_size_arg"); + goto failure; + if (size < min_value) { + err = size < 0 ? "too big" : "too small"; + goto failure; } return size; -} +failure: + snprintf(err_buf, sizeof err_buf, "--%s value is %s: %s\n", opt_name, err, size_arg); + return -1; +} static void create_refuse_error(int which) { @@ -1530,8 +1531,6 @@ int parse_arguments(int *argc_p, const char ***argv_p) if (daemon_filter_list.head) { int rej; char *cp = strdup(arg); - if (!cp) - out_of_memory("parse_arguments"); if (!*cp) rej = 1; else { @@ -1655,8 +1654,6 @@ int parse_arguments(int *argc_p, const char ***argv_p) remote_option_alloc += 16; remote_options = realloc_array(remote_options, const char *, remote_option_alloc); - if (!remote_options) - out_of_memory("parse_arguments"); if (!remote_option_cnt) remote_options[0] = "ARG0"; } @@ -1686,39 +1683,25 @@ int parse_arguments(int *argc_p, const char ***argv_p) break; case OPT_MAX_SIZE: - if ((max_size = parse_size_arg(&max_size_arg, 'b')) < 0) { - snprintf(err_buf, sizeof err_buf, - "--max-size value is invalid: %s\n", - max_size_arg); + if ((max_size = parse_size_arg(max_size_arg, 'b', "max-size", 0)) < 0) return 0; - } + max_size_arg = num_to_byte_string(max_size); break; case OPT_MIN_SIZE: - if ((min_size = parse_size_arg(&min_size_arg, 'b')) < 0) { - snprintf(err_buf, sizeof err_buf, - "--min-size value is invalid: %s\n", - min_size_arg); + if ((min_size = parse_size_arg(min_size_arg, 'b', "min-size", 0)) < 0) return 0; - } + min_size_arg = num_to_byte_string(min_size); break; - case OPT_BWLIMIT: - { - OFF_T limit = parse_size_arg(&bwlimit_arg, 'K'); - if (limit < 0) { - snprintf(err_buf, sizeof err_buf, - "--bwlimit value is invalid: %s\n", bwlimit_arg); - return 0; - } - bwlimit = (limit + 512) / 1024; - if (limit && !bwlimit) { - snprintf(err_buf, sizeof err_buf, - "--bwlimit value is too small: %s\n", bwlimit_arg); - return 0; - } - } + case OPT_BWLIMIT: { + ssize_t size = parse_size_arg(bwlimit_arg, 'K', "bwlimit", 512); + if (size < 0) + return 0; + bwlimit_arg = num_to_byte_string(size); + bwlimit = (size + 512) / 1024; break; + } case OPT_APPEND: if (am_server) @@ -1900,6 +1883,18 @@ int parse_arguments(int *argc_p, const char ***argv_p) exit_cleanup(0); } + if (!max_alloc_arg) { + max_alloc_arg = getenv("RSYNC_MAX_ALLOC"); + if (max_alloc_arg && !*max_alloc_arg) + max_alloc_arg = NULL; + } + if (max_alloc_arg) { + ssize_t size = parse_size_arg(max_alloc_arg, 'B', "max-alloc", 1024*1024); + if (size < 0) + return 0; + max_alloc = size; + } + if (protect_args < 0) { if (am_server) protect_args = 0; @@ -2770,6 +2765,11 @@ void server_options(char **args, int *argc_p) } } + if (max_alloc_arg && max_alloc != DEFAULT_MAX_ALLOC) { + args[ac++] = "--max-alloc"; + args[ac++] = max_alloc_arg; + } + /* --delete-missing-args needs the cooperation of both sides, but * the sender can handle --ignore-missing-args by itself. */ if (missing_args == 2) diff --git a/packaging/auto-Makefile b/packaging/auto-Makefile new file mode 100644 index 00000000..cc181536 --- /dev/null +++ b/packaging/auto-Makefile @@ -0,0 +1,11 @@ +TARGETS := all install install-ssl-daemon install-all install-strip conf gen gensend reconfigure restatus \ + proto man clean cleantests distclean test check check29 check30 installcheck splint doxygen doxygen-upload + +.PHONY: $(TARGETS) auto-prep + +$(TARGETS): auto-prep + make -C build $@ + +auto-prep: + @if test x`packaging/prep-auto-dir` = x; then echo "auto-build-save is not setup"; exit 1; fi + @echo 'Build branch: '`readlink build/.branch | tr % /` diff --git a/packaging/branch-from-patch b/packaging/branch-from-patch index dfd3a29a..440b5835 100755 --- a/packaging/branch-from-patch +++ b/packaging/branch-from-patch @@ -171,4 +171,4 @@ if __name__ == '__main__': args = parser.parse_args() main() -# vim: sw=4 et +# vim: sw=4 et ft=python diff --git a/packaging/lsb/rsync.spec b/packaging/lsb/rsync.spec index 22ae1ce5..939e4647 100644 --- a/packaging/lsb/rsync.spec +++ b/packaging/lsb/rsync.spec @@ -1,6 +1,6 @@ Summary: A fast, versatile, remote (and local) file-copying tool Name: rsync -Version: 3.2.1 +Version: 3.2.2 %define fullversion %{version} Release: 1 %define srcdir src @@ -79,8 +79,8 @@ rm -rf $RPM_BUILD_ROOT %dir /etc/rsync-ssl/certs %changelog -* Mon Jun 22 2020 Wayne Davison -Released 3.2.1. +* Sat Jul 04 2020 Wayne Davison +Released 3.2.2. * Fri Mar 21 2008 Wayne Davison Added installation of /etc/xinetd.d/rsync file and some commented-out diff --git a/packaging/md2html b/packaging/md2html index e4ee226a..0ee7c405 100755 --- a/packaging/md2html +++ b/packaging/md2html @@ -32,6 +32,22 @@ blockquote pre code { dd p:first-of-type { margin-block-start: 0em; } +table { + border-color: grey; + border-spacing: 0; +} +tr { + border-top: 1px solid grey; +} +tr:nth-child(2n) { + background-color: #f6f8fa; +} +th, td { + border: 1px solid #dfe2e5; + text-align: center; + padding-left: 1em; + padding-right: 1em; +} """ @@ -73,11 +89,8 @@ def main(): def html_via_cmarkgfm(txt): - return cmarkgfm.markdown_to_html(txt) - - -def html_via_commonmark(txt): - return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) + # Our NEWS.md file has a gfm table in it. + return cmarkgfm.github_flavored_markdown_to_html(txt) if __name__ == '__main__': @@ -90,10 +103,6 @@ if __name__ == '__main__': import cmarkgfm md_parser = html_via_cmarkgfm except: - try: - import commonmark - md_parser = html_via_commonmark - except: - die("Failed to find cmarkgfm or commonmark for python3.") + die("Failed to find cmarkgfm for python3.") main() diff --git a/packaging/patch-update b/packaging/patch-update index c6b51527..a387ad72 100755 --- a/packaging/patch-update +++ b/packaging/patch-update @@ -13,9 +13,9 @@ sys.path = ['packaging'] + sys.path from pkglib import * MAKE_GEN_CMDS = [ - 'make -f prepare-source.mak conf'.split(), - './config.status'.split(), - 'make gen'.split(), + './prepare-source'.split(), + 'cd build && if test -f config.status ; then ./config.status ; else ../configure ; fi', + 'make -C build gen'.split(), ] TMP_DIR = "patches.gen" @@ -33,6 +33,9 @@ def main(): master_commit = latest_git_hash(args.base_branch) + if cmd_txt_chk(['packaging/prep-auto-dir']) == '': + die('You must setup an auto-build-save dir to use this script.') + if args.gen: if os.path.lexists(TMP_DIR): die(f'"{TMP_DIR}" must not exist in the current directory.') @@ -42,7 +45,7 @@ def main(): cmd_chk(cmd) cmd_chk(['rsync', '-a', *gen_files, f'{TMP_DIR}/master/']) - last_touch = time.time() + last_touch = int(time.time()) # Start by finding all patches so that we can load all possible parents. patches = sorted(list(get_patch_branches(args.base_branch))) @@ -91,9 +94,10 @@ def main(): if args.gen: shutil.rmtree(TMP_DIR) - while last_touch >= time.time(): + while last_touch >= int(time.time()): time.sleep(1) cmd_chk(['git', 'checkout', starting_branch]) + cmd_chk(['packaging/prep-auto-dir'], discard='output') def update_patch(patch): @@ -113,31 +117,30 @@ def update_patch(patch): print(f"======== {patch} ========") - while args.gen and last_touch >= time.time(): + while args.gen and last_touch >= int(time.time()): time.sleep(1) - s = cmd_run(f"git checkout patch/{args.base_branch}/{patch}".split()) + + branch = f"patch/{args.base_branch}/{patch}" + s = cmd_run(['git', 'checkout', branch]) if s.returncode != 0: return 0 s = cmd_run(['git', 'merge', based_on]) ok = s.returncode == 0 - if not ok or args.shell: - m = re.search(r'([^/]+)$', parent) - parent_dir = m[1] - if not ok: - print(f'"git merge {based_on}" incomplete -- please fix.') - os.environ['PS1'] = f"[{parent_dir}] {patch}: " - while True: - s = cmd_run([os.environ.get('SHELL', '/bin/sh')]) - if s.returncode != 0: - ans = input("Abort? [n/y] ") - if re.match(r'^y', ans, flags=re.I): - return 0 - continue - cur_branch, is_clean, status_txt = check_git_status(0) - if is_clean: - break - print(status_txt, end='') + if not ok or args.cmd or args.shell: + cmd_chk(['packaging/prep-auto-dir'], discard='output') + if not ok: + print(f'"git merge {based_on}" incomplete -- please fix.') + if not run_a_shell(parent, patch): + return 0 + if args.cmd: + if cmd_run(args.cmd).returncode != 0: + if not run_a_shell(parent, patch): + return 0 + ok = False + if ok and args.shell: + if not run_a_shell(parent, patch): + return 0 with open(f"{args.patches_dir}/{patch}.diff", 'w', encoding='utf-8') as fh: fh.write(description[patch]) @@ -150,7 +153,7 @@ def update_patch(patch): cmd_chk(['rsync', '-a', *gen_files, f"{TMP_DIR}/{patch}/"]) else: gen_files = [ ] - last_touch = time.time() + last_touch = int(time.time()) proc = cmd_pipe(['git', 'diff', based_on]) skipping = False @@ -185,17 +188,38 @@ def update_patch(patch): line = plus_re.sub(r'+++ b/\1', line) fh.write(line) proc.communicate() - for fn in gen_files: - os.unlink(fn) return 1 +def run_a_shell(parent, patch): + m = re.search(r'([^/]+)$', parent) + parent_dir = m[1] + os.environ['PS1'] = f"[{parent_dir}] {patch}: " + + while True: + s = cmd_run([os.environ.get('SHELL', '/bin/sh')]) + if s.returncode != 0: + ans = input("Abort? [n/y] ") + if re.match(r'^y', ans, flags=re.I): + return False + continue + cur_branch, is_clean, status_txt = check_git_status(0) + if is_clean: + break + print(status_txt, end='') + + cmd_run('rm -f build/*.o build/*/*.o') + + return True + + if __name__ == '__main__': parser = argparse.ArgumentParser(description="Turn a git branch back into a diff files in the patches dir.", add_help=False) parser.add_argument('--branch', '-b', dest='base_branch', metavar='BASE_BRANCH', default='master', help="The branch the patch is based on. Default: master.") parser.add_argument('--skip-check', action='store_true', help="Skip the check that ensures starting with a clean branch.") parser.add_argument('--shell', '-s', action='store_true', help="Launch a shell for every patch/BASE/* branch updated, not just when a conflict occurs.") + parser.add_argument('--cmd', '-c', help="Run a command in every patch branch.") parser.add_argument('--gen', metavar='DIR', nargs='?', const='', help='Include generated files. Optional DIR value overrides the default of using the "patches" dir.') parser.add_argument('--patches-dir', '-p', metavar='DIR', default='patches', help="Override the location of the rsync-patches dir. Default: patches.") parser.add_argument('patch_files', metavar='patches/DIFF_FILE', nargs='*', help="Specify what patch diff files to process. Default: all of them.") @@ -207,4 +231,4 @@ if __name__ == '__main__': args.patches_dir = args.gen main() -# vim: sw=4 et +# vim: sw=4 et ft=python diff --git a/packaging/pkglib.py b/packaging/pkglib.py index 628669b0..b201bf77 100644 --- a/packaging/pkglib.py +++ b/packaging/pkglib.py @@ -181,38 +181,43 @@ def mandate_gensend_hook(): # Snag the GENFILES values out of the Makefile.in file and return them as a list. -def get_gen_files(): +def get_gen_files(want_dir_plus_list=False): cont_re = re.compile(r'\\\n') - extras = [ ] + gen_files = [ ] + + auto_dir = os.path.join('auto-build-save', cmd_txt('git rev-parse --abbrev-ref HEAD').strip().replace('/', '%')) with open('Makefile.in', 'r', encoding='utf-8') as fh: for line in fh: - if not extras: + if not gen_files: chk = re.sub(r'^GENFILES=', '', line) if line == chk: continue line = chk m = re.search(r'\\$', line) line = re.sub(r'^\s+|\s*\\\n?$|\s+$', '', line) - extras += line.split() + gen_files += line.split() if not m: break - return extras + if want_dir_plus_list: + return (auto_dir, gen_files) + return [ os.path.join(auto_dir, fn) for fn in gen_files ] -def get_configure_version(): - with open('configure.ac', 'r', encoding='utf-8') as fh: - for line in fh: - m = re.match(r'^AC_INIT\(\[rsync\],\s*\[(\d.+?)\]', line) - if m: - return m[1] - die("Unable to find AC_INIT with version in configure.ac") + +def get_rsync_version(): + with open('version.h', 'r', encoding='utf-8') as fh: + txt = fh.read() + m = re.match(r'^#define\s+RSYNC_VERSION\s+"(\d.+?)"', txt) + if m: + return m[1] + die("Unable to find RSYNC_VERSION define in version.h") def get_NEWS_version_info(): - rel_re = re.compile(r'^\| \d{2} \w{3} \d{4}\s+\|\s+(?P\d+\.\d+\.\d+)\s+\|\s+(?P\d{2} \w{3} \d{4}\s+)?\|\s+(?P\d+)\s+\|') + rel_re = re.compile(r'^\| \S{2} \w{3} \d{4}\s+\|\s+(?P\d+\.\d+\.\d+)\s+\|\s+(?P\d{2} \w{3} \d{4})?\s+\|\s+(?P\d+)\s+\|') last_version = last_protocol_version = None pdate = { } @@ -228,12 +233,11 @@ def get_NEWS_version_info(): pdate[m['ver']] = m['pdate'] if m['ver'] == last_version: last_protocol_version = m['pver'] - break if not last_protocol_version: die(f"Unable to determine protocol_version for {last_version}.") - return last_version, last_protocol_version + return last_version, last_protocol_version, pdate def get_protocol_versions(): diff --git a/packaging/prep-auto-dir b/packaging/prep-auto-dir new file mode 100755 index 00000000..a9516df8 --- /dev/null +++ b/packaging/prep-auto-dir @@ -0,0 +1,39 @@ +#!/bin/sh -e + +# This script will setup the build dir based on the current git branch and the +# directory auto-build-save/$BRANCH. We don't use a symlink for the build dir +# because we want to maximize the ccache reuse, so all builds must happen in +# the same real dir. When a dir is moved out of auto-build-save/$BRANCH to the +# build dir, it is replaced with a symlink so that it can still be found under +# that dir. The build dir also gets a .branch -> $BRANCH symlink so that we +# can figure out the current build dir's branch. + +# To get started, just clone the rsync git repo and create the auto-build-save +# dir. If you have an existing git checkout and it is not in a pristine state, +# run "make distclean" before creating the auto-build-save dir. + +auto_top='auto-build-save' +if test -d $auto_top -a -d .git; then + desired_branch=`git rev-parse --abbrev-ref HEAD | tr / %` + auto_dir="$auto_top/$desired_branch" + if test -d build; then + cur_branch=`readlink build/.branch` + else + cur_branch='/' + fi + if test "$desired_branch" != "$cur_branch"; then + if test "$cur_branch" != /; then + rm -f "$auto_top/$cur_branch" + mv build "$auto_top/$cur_branch" + fi + test -d "$auto_dir" || mkdir "$auto_dir" + test -h "$auto_dir/.branch" || ln -s "$desired_branch" "$auto_dir/.branch" + mv "$auto_dir" build + ln -s ../build "$auto_dir" + fi + if test ! -h Makefile; then + rm -f Makefile + ln -s packaging/auto-Makefile Makefile + fi + echo $desired_branch +fi diff --git a/packaging/release-rsync b/packaging/release-rsync index 3afa79bd..d1a3f590 100755 --- a/packaging/release-rsync +++ b/packaging/release-rsync @@ -13,6 +13,7 @@ sys.path = ['packaging'] + sys.path from pkglib import * +os.environ['LESS'] = 'mqeiXR'; # Make sure that -F is turned off and -R is turned on. dest = os.environ['HOME'] + '/samba-rsync-ftp' ORIGINAL_PATH = os.environ['PATH'] @@ -29,7 +30,11 @@ def main(): signal.signal(signal.SIGINT, signal_handler) - gen_files = get_gen_files() + if cmd_txt_chk(['packaging/prep-auto-dir']) == '': + die('You must setup an auto-build-save dir to use this script.'); + + auto_dir, gen_files = get_gen_files(True) + gen_pathnames = [ os.path.join(auto_dir, fn) for fn in gen_files ] dash_line = '=' * 74 @@ -52,13 +57,13 @@ def main(): check_git_state(args.master_branch, True, 'patches') - confversion = get_configure_version() + curversion = get_rsync_version() # All version values are strings! - lastversion, last_protocol_version = get_NEWS_version_info() + lastversion, last_protocol_version, pdate = get_NEWS_version_info() protocol_version, subprotocol_version = get_protocol_versions() - version = confversion + version = curversion m = re.search(r'pre(\d+)', version) if m: version = re.sub(r'pre\d+', 'pre' + str(int(m[1]) + 1), version) @@ -88,8 +93,8 @@ def main(): cmd_chk(['git', 'tag', '-d', v_ver]) version = re.sub(r'[-.]*pre[-.]*', 'pre', version) - if 'pre' in version and not confversion.endswith('dev'): - lastversion = confversion + if 'pre' in version and not curversion.endswith('dev'): + lastversion = curversion ans = input(f"Enter the previous version to produce a patch against: [{lastversion}] ") if ans != '': @@ -111,11 +116,8 @@ def main(): release += '.' + pre finalversion = re.sub(r'pre\d+', '', version) - if protocol_version == last_protocol_version: - proto_changed = 'unchanged' - proto_change_date = ' ' * 11 - else: - proto_changed = 'changed' + proto_changed = protocol_version != last_protocol_version + if proto_changed: if finalversion in pdate: proto_change_date = pdate[finalversion] else: @@ -124,6 +126,8 @@ def main(): if re.match(r'^\d\d \w\w\w \d\d\d\d$', ans): break proto_change_date = ans + else: + proto_change_date = ' ' * 11 if 'pre' in lastversion: if not pre: @@ -152,7 +156,7 @@ release is "{release}" About to: - tweak SUBPROTOCOL_VERSION in rsync.h, if needed - - tweak the version in configure.ac and the spec files + - tweak the version in version.h and the spec files - tweak NEWS.md to ensure header values are correct - generate configure.sh, config.h.in, and proto.h - page through the differences @@ -167,16 +171,15 @@ About to: '%define srcdir': srcdir, } - tweak_files = 'configure.ac rsync.h NEWS.md'.split() + tweak_files = 'version.h rsync.h NEWS.md'.split() tweak_files += glob.glob('packaging/*.spec') tweak_files += glob.glob('packaging/*/*.spec') for fn in tweak_files: with open(fn, 'r', encoding='utf-8') as fh: old_txt = txt = fh.read() - if 'configure' in fn: - x_re = re.compile(r'^(AC_INIT\(\[rsync\],\s*\[)\d.+?(\])', re.M) - txt = replace_or_die(x_re, r'\g<1>%s\2' % version, txt, f"Unable to update AC_INIT with version in {fn}") + if fn == 'version.h': + txt = f'#define RSYNC_VERSION "{version}"\n' elif '.spec' in fn: for var, val in specvars.items(): x_re = re.compile(r'^%s .*' % re.escape(var), re.M) @@ -185,14 +188,17 @@ About to: txt = replace_or_die(x_re, r'%s \1' % cl_today, txt, f"Unable to update ChangeLog header in {fn}") elif fn == 'rsync.h': x_re = re.compile('(#define\s+SUBPROTOCOL_VERSION)\s+(\d+)') - repl = lambda m: m[1] + ' ' + '0' if not pre or proto_changed != 'changed' else 1 if m[2] == '0' else m[2] + repl = lambda m: m[1] + ' ' + ('0' if not pre or proto_changed else 1 if m[2] == '0' else m[2]) txt = replace_or_die(x_re, repl, txt, f"Unable to find SUBPROTOCOL_VERSION define in {fn}") elif fn == 'NEWS.md': efv = re.escape(finalversion) - x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+Protocol: .+\n' % efv) + x_re = re.compile(r'^<.+>\s+# NEWS for rsync %s \(UNRELEASED\)\s+## Changes in this version:\n' % efv + + r'(\n### PROTOCOL NUMBER:\s+- The protocol number was changed to \d+\.\n)?') rel_day = 'UNRELEASED' if pre else today repl = (f'\n\n# NEWS for rsync {finalversion} ({rel_day})\n\n' - + f"Protocol: {protocol_version} ({proto_changed})\n") + + '## Changes in this version:\n') + if proto_changed: + repl += f'\n### PROTOCOL NUMBER:\n\n - The protocol number was changed to {protocol_version}.\n' good_top = re.sub(r'\(.*?\)', '(UNRELEASED)', repl, 1) msg = f"The top lines of {fn} are not in the right format. It should be:\n" + good_top txt = replace_or_die(x_re, repl, txt, msg) @@ -215,10 +221,10 @@ About to: srctar_name = f"{rsync_ver}.tar.gz" pattar_name = f"rsync-patches-{version}.tar.gz" diff_name = f"{rsync_lastver}-{version}.diffs.gz" - srctar_file = f"{dest}/{srcdir}/{srctar_name}" - pattar_file = f"{dest}/{srcdir}/{pattar_name}" - diff_file = f"{dest}/{srcdiffdir}/{diff_name}" - lasttar_file = f"{dest}/{lastsrcdir}/{rsync_lastver}.tar.gz" + srctar_file = os.path.join(dest, srcdir, srctar_name) + pattar_file = os.path.join(dest, srcdir, pattar_name) + diff_file = os.path.join(dest, srcdiffdir, diff_name) + lasttar_file = os.path.join(dest, lastsrcdir, rsync_lastver + '.tar.gz') print(f"""\ {dash_line} @@ -236,8 +242,7 @@ About to: if s.returncode: die('Aborting') - cmd_chk('make reconfigure ; make gen') - cmd_chk(['rsync', '-a', *gen_files, 'SaVeDiR/']) + cmd_chk('make gen') print(f'Creating any missing patch branches.') s = cmd_run(f'packaging/branch-from-patch --branch={args.master_branch} --add-missing') @@ -253,9 +258,6 @@ About to: print(f'\nVisiting all "patch/{args.master_branch}/*" branches ...') cmd_run(f"packaging/patch-update --branch={args.master_branch} --skip-check --shell") - cmd_run("rm -f *.[o15] *.html") - cmd_chk('rsync -a SaVeDiR/ .'.split()) - if os.path.isdir('patches/.git'): s = cmd_run(f"cd patches && git commit -a -m 'The patches for {version}.'") if s.returncode: @@ -294,12 +296,12 @@ About to: os.environ['PATH'] = ORIGINAL_PATH # Extract the generated files from the old tar. - tweaked_gen_files = [ f"{rsync_lastver}/{x}" for x in gen_files ] + tweaked_gen_files = [ os.path.join(rsync_lastver, fn) for fn in gen_files ] cmd_run(['tar', 'xzf', lasttar_file, *tweaked_gen_files]) os.rename(rsync_lastver, 'a') print(f"Creating {diff_file} ...") - cmd_chk(['rsync', '-a', *gen_files, 'b/']) + cmd_chk(['rsync', '-a', *gen_pathnames, 'b/']) sed_script = r's:^((---|\+\+\+) [ab]/[^\t]+)\t.*:\1:' # CAUTION: must not contain any single quotes! cmd_chk(f"(git diff v{lastversion} {v_ver} -- ':!.github'; diff -upN a b | sed -r '{sed_script}') | gzip -9 >{diff_file}") @@ -317,18 +319,13 @@ About to: os.mkdir(f"{rsync_ver}/patches", 0o755) cmd_chk(f"packaging/patch-update --skip-check --branch={args.master_branch} --gen={rsync_ver}/patches".split()) - cmd_run("rm -f *.[o15] *.html") - cmd_chk('rsync -a SaVeDiR/ .'.split()) - shutil.rmtree('SaVeDiR') - cmd_chk('make gen'.split()) - print(f"Creating {pattar_file} ...") cmd_chk(['fakeroot', 'tar', 'chzf', pattar_file, rsync_ver + '/patches']) shutil.rmtree(rsync_ver) print(f"Updating the other files in {dest} ...") md_files = 'README.md NEWS.md'.split() - html_files = [ fn for fn in gen_files if fn.endswith('.html') ] + html_files = [ fn for fn in gen_pathnames if fn.endswith('.html') ] cmd_chk(['rsync', '-a', *md_files, *html_files, dest]) cmd_chk(["packaging/md2html"] + [ dest +'/'+ fn for fn in md_files ]) @@ -381,4 +378,4 @@ if __name__ == '__main__': args = parser.parse_args() main() -# vim: sw=4 et +# vim: sw=4 et ft=python diff --git a/packaging/systemd/rsync.service b/packaging/systemd/rsync.service index fafe72b3..8a0b5820 100644 --- a/packaging/systemd/rsync.service +++ b/packaging/systemd/rsync.service @@ -16,14 +16,14 @@ RestartSec=1 # This is generally used for public file distribution, [...] # # So let's assume some extra security is more than welcome here. We do full -# system protection (which makes it read-only) and hide users' homes and +# system protection (which makes /usr, /boot, & /etc read-only) and hide # devices. To override these defaults, it's best to do so in the drop-in # directory, often done via `systemctl edit rsync.service`. The file needs # just the bare minimum of the right [heading] and override values. # See systemd.unit(5) and search for "drop-in" for full details. ProtectSystem=full -ProtectHome=on +#ProtectHome=on|off|read-only PrivateDevices=on NoNewPrivileges=on diff --git a/packaging/systemd/rsync@.service b/packaging/systemd/rsync@.service index e1f48da8..63ba0c7c 100644 --- a/packaging/systemd/rsync@.service +++ b/packaging/systemd/rsync@.service @@ -16,13 +16,13 @@ StandardError=journal # This is generally used for public file distribution, [...] # # So let's assume some extra security is more than welcome here. We do full -# system protection (which makes it read-only) and hide users' homes and +# system protection (which makes /usr, /boot, & /etc read-only) and hide # devices. To override these defaults, it's best to do so in the drop-in # directory, often done via `systemctl edit rsync@.service`. The file needs # just the bare minimum of the right [heading] and override values. # See systemd.unit(5) and search for "drop-in" for full details. ProtectSystem=full -ProtectHome=on +#ProtectHome=on|off|read-only PrivateDevices=on NoNewPrivileges=on diff --git a/packaging/var-checker b/packaging/var-checker index e021127c..fe7491df 100755 --- a/packaging/var-checker +++ b/packaging/var-checker @@ -84,4 +84,4 @@ if __name__ == '__main__': args = parser.parse_args() main() -# vim: sw=4 et +# vim: sw=4 et ft=python diff --git a/params.c b/params.c index 910ef233..da8a8f88 100644 --- a/params.c +++ b/params.c @@ -212,11 +212,6 @@ static BOOL Section( FILE *InFile, BOOL (*sfunc)(char *) ) { bSize += BUFR_INC; bufr = realloc_array( bufr, char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s Memory re-allocation failure.", func); - return( False ); - } } /* Handle a single character. */ @@ -306,11 +301,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c ) { bSize += BUFR_INC; bufr = realloc_array( bufr, char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s Memory re-allocation failure.", func) ; - return( False ); - } } switch( c ) @@ -382,11 +372,6 @@ static BOOL Parameter( FILE *InFile, BOOL (*pfunc)(char *, char *), int c ) { bSize += BUFR_INC; bufr = realloc_array( bufr, char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s Memory re-allocation failure.", func) ; - return( False ); - } } switch( c ) @@ -639,12 +624,6 @@ int pm_process( char *FileName, { /* allocate one, then parse, */ bSize = BUFR_INC; /* then free. */ bufr = new_array( char, bSize ); - if( NULL == bufr ) - { - rprintf(FLOG, "%s memory allocation failure.\n", func); - fclose(InFile); - return( False ); - } result = Parse( InFile, sfunc, pfunc ); free( bufr ); bufr = NULL; diff --git a/prepare-source b/prepare-source index a41a66a3..92763508 100755 --- a/prepare-source +++ b/prepare-source @@ -16,6 +16,26 @@ if test x"$dir" = x; then dir=. fi +if test "$dir" = '.'; then + branch=`packaging/prep-auto-dir` || exit 1 + if test x"$branch" != x; then + cd build || exit 1 + dir=.. + fi +fi + +if test "$dir" != '.'; then + for lnk in configure.ac m4; do + if test ! -h $lnk; then + rm -f $lnk # Just in case + ln -s "$dir/$lnk" $lnk + fi + done + for fn in configure.sh config.h.in aclocal.m4; do + test ! -f $fn -a -f "$dir/$fn" && cp -p "$dir/$fn" $fn + done +fi + if test $# = 0; then set -- build fi @@ -23,13 +43,16 @@ fi for action in "${@}"; do case "$action" in build|make) - (cd $dir && make -f prepare-source.mak) + make -f "$dir/prepare-source.mak" ;; fetch|fetchgen) - $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[ca]*' $dir if test "$action" = fetchgen; then - $dir/rsync-ssl -iip --no-motd rsync://download.samba.org/rsyncftp/generated-files/'[^ca]*' . + match='*' + else + match='[ca]*' fi + $dir/rsync-ssl -iipc --no-motd "rsync://download.samba.org/rsyncftp/generated-files/$match" ./ + test $? != 0 && continue sleep 1 # The following files need to be newer than aclocal.m4 touch configure.sh config.h.in ;; @@ -39,6 +62,7 @@ for action in "${@}"; do *) echo "Unknown action: $action" exit 1 + ;; esac if test $? = 0; then exit diff --git a/prepare-source.mak b/prepare-source.mak index bc1938de..08b916de 100644 --- a/prepare-source.mak +++ b/prepare-source.mak @@ -1,3 +1,6 @@ +SHELL=/bin/sh + +.PHONY: conf conf: configure.sh config.h.in aclocal.m4: m4/*.m4 diff --git a/rsync.1.md b/rsync.1.md index 9d9dfbae..76b55367 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -403,6 +403,7 @@ detailed description below for a complete description. --max-delete=NUM don't delete more than NUM files --max-size=SIZE don't transfer any file larger than SIZE --min-size=SIZE don't transfer any file smaller than SIZE +--max-alloc=SIZE change a limit relating to memory alloc --partial keep partially transferred files --partial-dir=DIR put a partially transferred file into DIR --delay-updates put all updated files into place at end @@ -520,9 +521,6 @@ your home directory (remove the '=' for that). list of compression algorithms, a list of compiled-in capabilities, a link to the rsync web site, and some license/copyright info. - Repeat the option (`-VV`) to include some optimization info at the end of - the capabilities list. - 0. `--verbose`, `-v` This option increases the amount of information you are given during the @@ -605,7 +603,7 @@ your home directory (remove the '=' for that). helps to avoid overpopulating the protocol data with extra message data. The option does not affect the remote side of a transfer without using - `--remote-option` -- e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`. + `--remote-option`, e.g. `-M--msgs2stderr` or `{-M,}--msgs2stderr`. Also keep in mind that connecting to a normal (non-remote-shell) daemon does not have a stderr channel to send messages back to the client side, so @@ -690,7 +688,7 @@ your home directory (remove the '=' for that). before-the-transfer "Does this file need to be updated?" check. The checksum used is auto-negotiated between the client and the server, but - can be overridden using either the `--checksum-choice` option or an + can be overridden using either the `--checksum-choice` (`--cc`) option or an environment variable that is discussed in that option's section. 0. `--archive`, `-a` @@ -1473,13 +1471,16 @@ your home directory (remove the '=' for that). The checksum options that you may be able to use are: - - `auto` (the default) - - `xxh64` (aka xxhash) + - `auto` (the default automatic choice) + - `xxh128` + - `xxh3` + - `xxh64` (aka `xxhash`) - `md5` - `md4` - `none` - Run `rsync -V` to see the default checksum list compiled into your version. + Run `rsync --version` to see the default checksum list compiled into your + version (which may differ from the list above). If "none" is specified for the first (or only) name, the `--whole-file` option is forced on and no checksum verification is performed on the @@ -1489,22 +1490,22 @@ your home directory (remove the '=' for that). The "auto" option is the default, where rsync bases its algorithm choice on a negotiation between the client and the server as follows: - If both the client and the server are at least version 3.2.0, they will - exchange a list of checksum names and choose the first one in the list that - they have in common. This typically means that they will choose xxh64 if - they both support it and fall back to MD5. If one side of the transfer is - not new enough to support this checksum negotiation, then a value is chosen - based on the protocol version (which chooses between MD5 and various - flavors of MD4 based on protocol age). + When both sides of the transfer are at least 3.2.0, rsync chooses the first + algorithm in the client's list of choices that is also in the server's list + of choices. If no common checksum choice is found, rsync exits with + an error. If the remote rsync is too old to support checksum negotiation, + a value is chosen based on the protocol version (which chooses between MD5 + and various flavors of MD4 based on protocol age). - You can also override the checksum using the RSYNC_CHECKSUM_LIST - environment variable by setting it to a space-separated list of checksum - names that you consider acceptable. If no common checksum is found, the - client exits with an error. This method does not allow you to specify the - transfer checksum separately from the pre-transfer checksum, and it ignores - "auto" and all unknown checksum names. If the remote rsync is not new - enough to handle a checksum negotiation list, the list is silently ignored - unless it contains the string "FAIL". + The default order can be customized by setting the environment variable + RSYNC_CHECKSUM_LIST to a space-separated list of acceptable checksum names. + If the string contains a "`&`" character, it is separated into the "client + string & server string", otherwise the same string + applies to both. If the string (or string portion) contains no + non-whitespace characters, the default checksum list is used. This method + does not allow you to specify the transfer checksum separately from the + pre-transfer checksum, and it discards "auto" and all unknown checksum + names. A list with only invalid names results in a failed negotiation. The use of the `--checksum-choice` option overrides this environment list. @@ -1731,12 +1732,16 @@ your home directory (remove the '=' for that). data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. - The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or - "MiB") is a mebibyte (1024\*1024), and "G" (or "GiB") is a gibibyte - (1024\*1024\*1024). If you want the multiplier to be 1000 instead of 1024, - use "KB", "MB", or "GB". (Note: lower-case is also accepted for all - values.) Finally, if the suffix ends in either "+1" or "-1", the value will - be offset by one byte in the indicated direction. + The accepted suffix letters are: `B`, `K`, `G`, `T`, and `P` for bytes, + kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes, + terabytes/tebibytes, and petabytes/pebibytes. If you use a single-char + suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are + multiples of 1024. If you use a two-letter suffix that ends with a "B" + (e.g. "kb") then you get units that are multiples of 1000. The suffix + letters can be any mix of upper and lower-case that you want to use. + + Finally, if the string ends with either "+1" or "-1", it is offset by one + byte in the indicated direction. The largest possible value is `8192P-1`. Examples: `--max-size=1.5mb-1` is 1499999 bytes, and `--max-size=2g+1` is 2147483649 bytes. @@ -1751,6 +1756,28 @@ your home directory (remove the '=' for that). Note that rsync versions prior to 3.1.0 did not allow `--min-size=0`. +0. `--max-alloc=SIZE` + + By default rsync limits an individual malloc/realloc to about 1GB in size. + For most people this limit works just fine and prevents a protocol error + causing rsync to request massive amounts of memory. However, if you have + many millions of files in a transfer, a large amount of server memory, and + you don't want to split up your transfer into multiple parts, you can + increase the per-allocation limit to something larger and rsync will + consume more memory. + + Keep in mind that this is not a limit on the total size of allocated + memory. It is a sanity-check value for each individual allocation. + + See the `--max-size` option for a description of how SIZE can be specified. + The default suffix if none is given is bytes. + + You can set a default value using the environment variable RSYNC_MAX_ALLOC + using the same SIZE values as supported by this option. If the remote + rsync doesn't understand the `--max-alloc` option, you can override an + environmental value by specifying `--max-alloc=1g`, which will make rsync + avoid sending the option to the remote side (because "1G" is the default). + 0. `--block-size=SIZE`, `-B` This forces the block size used in rsync's delta-transfer algorithm to a @@ -2073,8 +2100,9 @@ your home directory (remove the '=' for that). Rsync can also be configured (at build time) to have this option enabled by default (with is overridden by both the environment and the command-line). - Run `rsync -V` to check if this is the case, as it will display "default - protect-args" or "optional protect-args" depending on how it was compiled. + Run `rsync --version` to check if this is the case, as it will display + "default protect-args" or "optional protect-args" depending on how it was + compiled. This option will eventually become a new default setting at some as-yet-undetermined point in the future. @@ -2260,29 +2288,42 @@ your home directory (remove the '=' for that). destination machine, which reduces the amount of data being transmitted -- something that is useful over a slow connection. - The "zlib" compression method typically achieves better compression ratios - than can be achieved by using a compressing remote shell or a compressing - transport because it takes advantage of the implicit information in the - matching data blocks that are not explicitly sent over the connection. - This matching-data compression comes at a cost of CPU, though, and can be - disabled by using the "zlibx" compresson method instead. This can be - selected by repeating the `-z` option or specifying - `--compress-choice=zlibx`, but it only works if both sides of the transfer - are at least version 3.1.1. + Rsync supports multiple compression methods and will choose one for you + unless you force the choice using the `--compress-choice` (`--zc`) option. - Note that if you see an error about an option named `--old-compress` or - `--new-compress`, this is rsync trying to send the `--compress-choice=zlib` - or `--compress-choice=zlibx` option in a backward-compatible manner that - more rsync versions understand. This error indicates that the older rsync - version will not allow you to force the compression type. + Run `rsync --version` to see the default compress list compiled into your + version. - See the `--skip-compress` option for the default list of file suffixes that - will not be compressed. + When both sides of the transfer are at least 3.2.0, rsync chooses the first + algorithm in the client's list of choices that is also in the server's list + of choices. If no common compress choice is found, rsync exits with + an error. If the remote rsync is too old to support checksum negotiation, + its list is assumed to be "zlib". + + The default order can be customized by setting the environment variable + RSYNC_COMPRESS_LIST to a space-separated list of acceptable compression + names. If the string contains a "`&`" character, it is separated into the + "client string & server string", otherwise the same string applies to both. + If the string (or string portion) contains no + non-whitespace characters, the default compress list is used. Any unknown + compression names are discarded from the list, but a list with only invalid + names results in a failed negotiation. + + There are some older rsync versions that were configured to reject a `-z` + option and require the use of `-zz` because their compression library was + not compatible with the default zlib compression method. You can usually + ignore this weirdness unless the rsync server complains and tells you to + specify `-zz`. + + See also the `--skip-compress` option for the default list of file suffixes + that will trasnferred with no (or minimal) compression. 0. `--compress-choice=STR`, `--zc=STR` - This option can be used to override the automatic selection of the - compression algorithm that is the default when `--compress` is used. + This option can be used to override the automatic negotiation of the + compression algorithm that occurs when `--compress` is used. The option + implies `--compress` unless "none" was specified, which instead implies + `--no-compress`. The compression options that you may be able to use are: @@ -2292,25 +2333,18 @@ your home directory (remove the '=' for that). - `zlib` - `none` - Run `rsync -V` to see the compress list compiled into your version. - - The "zlibx" algorithm is given preference over "zlib" if both sides of the - transfer are at least version 3.2.0, otherwise it will choose "zlib" unless - you override it via something like `-zz`. These 2 algorithms are the stame - except that "zlibx" does not try to include matched data that was not - transferred in the compression computations. - - If "none" is specified, that is equivalent to using `--no-compress`. - - This option implies `--compress` unless "none" was specified. - - You can also override the compression negotiation using the - RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated - list of compression names that you consider acceptable. If no common - compress choice is found, the client exits with an error. It ignores - "auto" and all unknown compression names. If the remote rsync is not new - enough to handle a compression negotiation list, the list is silently - ignored unless it contains the string "FAIL". + Run `rsync --version` to see the default compress list compiled into your + version (which may differ from the list above). + + Note that if you see an error about an option named `--old-compress` or + `--new-compress`, this is rsync trying to send the `--compress-choice=zlib` + or `--compress-choice=zlibx` option in a backward-compatible manner that + more rsync versions understand. This error indicates that the older rsync + version on the server will not allow you to force the compression type. + + Note that the "zlibx" compression algorithm is just the "zlib" algorithm + with matched data excluded from the compression stream (to try to make it + more compatible with an external zlib implementation). 0. `--compress-level=NUM`, `--zl=NUM` @@ -2321,8 +2355,8 @@ your home directory (remove the '=' for that). "off"). The level values vary depending on the checksum in effect. Because rsync - will negotiate a checksum choice by default when the remote rsync is new - enough, it can be good to combine this option with a `--compress-choice` + will negotiate a checksum choice by default (when the remote rsync is new + enough), it can be good to combine this option with a `--compress-choice` (`--zc`) option unless you're sure of the choice in effect. For example: > rsync -aiv --zc=zstd --zl=22 host:src/ dest/ @@ -2352,8 +2386,10 @@ your home directory (remove the '=' for that). possible. Rsync sets the compression level on a per-file basis based on the file's suffix. If the compression algorithm has an "off" level (such as zlib/zlibx) then no compression occurs for those files. Other - algorithms have the level minimized to reduces the CPU usage as much as - possible. + algorithms that support changing the streaming level on-the-fly will have + the level minimized to reduces the CPU usage as much as possible for a + matching file. At this time, only zlib & zlibx compression support this + changing of levels on a per-file basis. The **LIST** should be one or more file suffixes (without the dot) separated by slashes (`/`). You may specify an empty string to indicate that no files @@ -3077,6 +3113,11 @@ your home directory (remove the '=' for that). with `--read-batch`. See the "BATCH MODE" section for details, and also the `--only-write-batch` option. + This option overrides the negotiated checksum & compress lists and always + negotiates a choice based on old-school md5/md4/zlib choices. If you want + a more modern choice, use the `--checksum-choice` (`--cc`) and/or + `--compress-choice` (`--zc`) options. + 0. `--only-write-batch=FILE` Works like `--write-batch`, except that no updates are made on the @@ -3157,8 +3198,8 @@ your home directory (remove the '=' for that). These options also exist in the `--daemon` mode section. If rsync was complied without support for IPv6, the `--ipv6` option will - have no effect. The `rsync -V` output will contain "`no IPv6`" if is the - case. + have no effect. The `rsync --version` output will contain "`no IPv6`" if + is the case. 0. `--checksum-seed=NUM` @@ -3271,8 +3312,8 @@ The options allowed when starting an rsync daemon are as follows: These options also exist in the regular rsync options section. If rsync was complied without support for IPv6, the `--ipv6` option will - have no effect. The `rsync -V` output will contain "`no IPv6`" if is the - case. + have no effect. The `rsync --version` output will contain "`no IPv6`" if + is the case. 0. `--help`, `-h` diff --git a/rsync.h b/rsync.h index 8af83ecf..41c4052b 100644 --- a/rsync.h +++ b/rsync.h @@ -20,6 +20,7 @@ #define False 0 #define True 1 +#define Unset (-1) /* Our BOOL values are always an int. */ #define BLOCK_SIZE 700 #define RSYNC_RSH_ENV "RSYNC_RSH" @@ -297,6 +298,7 @@ enum delret { #include "errcode.h" #include "config.h" +#include "version.h" /* The default RSYNC_RSH is always set in config.h. */ @@ -1078,6 +1080,9 @@ typedef struct { #define CPRES_LZ4 3 #define CPRES_ZSTD 4 +#define NSTR_CHECKSUM 0 +#define NSTR_COMPRESS 1 + struct name_num_item { int num; const char *name, *main_name; @@ -1262,12 +1267,19 @@ extern int errno; /* handler for null strings in printf format */ #define NS(s) ((s)?(s):"") +extern char *do_malloc; + /* Convenient wrappers for malloc and realloc. Use them. */ -#define new(type) ((type*)malloc(sizeof (type))) -#define new0(type) ((type*)calloc(1, sizeof (type))) -#define new_array(type, num) ((type*)_new_array((num), sizeof (type), 0)) -#define new_array0(type, num) ((type*)_new_array((num), sizeof (type), 1)) -#define realloc_array(ptr, type, num) ((type*)_realloc_array((ptr), sizeof(type), (num))) +#define new(type) ((type*)my_alloc(do_malloc, sizeof (type), 1, __FILE__, __LINE__)) +#define new0(type) ((type*)my_alloc(NULL, sizeof (type), 1, __FILE__, __LINE__)) +#define realloc_buf(ptr, num) my_alloc((ptr), (num), 1, __FILE__, __LINE__) + +#define new_array(type, num) ((type*)my_alloc(do_malloc, (num), sizeof (type), __FILE__, __LINE__)) +#define new_array0(type, num) ((type*)my_alloc(NULL, (num), sizeof (type), __FILE__, __LINE__)) +#define realloc_array(ptr, type, num) ((type*)my_alloc((ptr), (num), sizeof (type), __FILE__, __LINE__)) + +#undef strdup +#define strdup(s) my_strdup(s, __FILE__, __LINE__) /* use magic gcc attributes to catch format errors */ void rprintf(enum logcode , const char *, ...) diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md index bc5a571b..11f77cb0 100644 --- a/rsyncd.conf.5.md +++ b/rsyncd.conf.5.md @@ -424,6 +424,22 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details. Helpful hint: you probably want to specify "refuse options = delete" for a write-only module. +0. `open noatime` + + When set to True, this parameter tells the rsync daemon to open files with + the O_NOATIME flag + (on systems that support it) to avoid changing the access time of the files + that are being transferred. If your OS does not support the O_NOATIME flag + then rsync will silently ignore this option. Note also that some + filesystems are mounted to avoid updating the atime on read access even + without the O_NOATIME flag being set. + + When set to False, this parameters ensures that files on the server are not + opened with O_NOATIME. + + When set to Unset (the default) the user controls the setting via + `--open-noatime`. + 0. `list` This parameter determines whether this module is listed when the client @@ -905,15 +921,15 @@ the values of parameters. See the GLOBAL PARAMETERS section for more details. - `--server`: Required for rsync to even work. - `--rsh`, `-e`: Required to convey compatibility flags to the server. - - `--log-format`: This is required to convey things like - `--itemize-changes` to a remote receiver. Is an older name for - `--out-format` that is still passed to the server for improved backward - compatibility and should not be confused with `--log-file-format`. + - `--out-format`: This is required to convey output behavior to a remote + receiver. While rsync passes the older alias `--log-format` for + compatibility reasons, this options should not be confused with + `--log-file-format`. - `--sender`: Use "write only" parameter instead of refusing this. - `--dry-run`, `-n`: Who would want to disable this? - - `--protect-args`, `-n`: This actually makes transfers safer. - - `--from0`, `-0`: Make it easier to accept/refuse `--files-from` without - affecting this modifier. + - `--protect-args`, `-s`: This actually makes transfers safer. + - `--from0`, `-0`: Makes it easier to accept/refuse `--files-from` without + affecting this helpful modifier. - `--iconv`: This is auto-disabled based on "charset" parameter. - `--no-iconv`: Most transfers use this option. - `--checksum-seed`: Is a fairly rare, safe option. @@ -1105,7 +1121,7 @@ An example nginx proxy setup is as follows: > listen [::]:874 ssl; > > ssl_certificate /etc/letsencrypt/example.com/fullchain.pem; -> ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem +> ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem; > > proxy_pass localhost:873; > proxy_protocol on; # Requires "proxy protocol = true" diff --git a/sender.c b/sender.c index abd79bd2..94761c26 100644 --- a/sender.c +++ b/sender.c @@ -65,13 +65,10 @@ BOOL extra_flist_sending_enabled; **/ static struct sum_struct *receive_sums(int f) { - struct sum_struct *s; - int32 i; + struct sum_struct *s = new(struct sum_struct); int lull_mod = protocol_version >= 31 ? 0 : allowed_lull * 5; OFF_T offset = 0; - - if (!(s = new(struct sum_struct))) - out_of_memory("receive_sums"); + int32 i; read_sum_head(f, s); @@ -92,8 +89,7 @@ static struct sum_struct *receive_sums(int f) if (s->count == 0) return(s); - if (!(s->sums = new_array(struct sum_buf, s->count))) - out_of_memory("receive_sums"); + s->sums = new_array(struct sum_buf, s->count); for (i = 0; i < s->count; i++) { s->sums[i].sum1 = read_int(f); diff --git a/socket.c b/socket.c index ee6fb6c6..c2075adf 100644 --- a/socket.c +++ b/socket.c @@ -26,6 +26,7 @@ #include "rsync.h" #include "itypes.h" +#include "ifuncs.h" #ifdef HAVE_NETINET_IN_SYSTM_H #include #endif @@ -248,8 +249,6 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint) for (res = res0, addr_cnt = 0; res; res = res->ai_next, addr_cnt++) {} errnos = new_array0(int, addr_cnt); - if (!errnos) - out_of_memory("open_socket_out"); s = -1; /* Try to connect to all addresses for this machine until we get @@ -354,8 +353,7 @@ int open_socket_out_wrapped(char *host, int port, const char *bind_addr, int af_ len += hlen; } f = prog; - if (!(prog = new_array(char, len))) - out_of_memory("open_socket_out_wrapped"); + prog = new_array(char, len); for (t = prog; *f; f++) { if (*f == '%') { switch (*++f) { @@ -423,8 +421,6 @@ static int *open_socket_in(int type, int port, const char *bind_addr, socks = new_array(int, maxs + 1); errmsgs = new_array(char *, maxs); - if (!socks || !errmsgs) - out_of_memory("open_socket_in"); /* We may not be able to create the socket, if for example the * machine knows about IPv6 in the C library, but not in the @@ -684,9 +680,6 @@ void set_socket_options(int fd, char *options) options = strdup(options); - if (!options) - out_of_memory("set_socket_options"); - for (tok = strtok(options, " \t,"); tok; tok = strtok(NULL," \t,")) { int ret=0,i; int value = 1; diff --git a/support/rrsync b/support/rrsync index 34e62300..eabec3dc 100644 --- a/support/rrsync +++ b/support/rrsync @@ -111,6 +111,7 @@ our %long_opt = ( 'list-only' => 0, 'log-file' => $only eq 'r' ? -1 : 3, 'log-format' => 1, + 'max-alloc' => 1, 'max-delete' => 1, 'max-size' => 1, 'min-size' => 1, diff --git a/t_stub.c b/t_stub.c index 2748ca20..954b86da 100644 --- a/t_stub.c +++ b/t_stub.c @@ -33,6 +33,7 @@ int preserve_xattrs = 0; int preserve_perms = 0; int preserve_executability = 0; int open_noatime = 0; +size_t max_alloc = 1024*1024*1024; /* max_alloc is needed when combined with util2.o */ char *partial_dir; char *module_dir; filter_rule_list daemon_filter_list; diff --git a/testsuite/daemon.test b/testsuite/daemon.test index a8acd04e..a736ee83 100644 --- a/testsuite/daemon.test +++ b/testsuite/daemon.test @@ -91,12 +91,13 @@ drwxr-xr-x DIR ####/##/## ##:##:## foo EOT diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed" -$RSYNC -rU localhost::test-from/f* \ - | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \ - | tee "$outfile" -cat <"$chkfile" +if $RSYNC --version | grep "[, ] atimes" >/dev/null; then + $RSYNC -rU localhost::test-from/f* \ + | sed "$FILE_REPL" | sed "$DIR_REPL" | sed "$LS_REPL" \ + | tee "$outfile" + cat <"$chkfile" drwxr-xr-x DIR ####/##/## ##:##:## foo -rw-r--r-- 4 ####/##/## ##:##:## ####/##/## ##:##:## foo/one EOT -diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed" - + diff $diffopt "$chkfile" "$outfile" || test_fail "test 3 failed" +fi diff --git a/testsuite/itemize.test b/testsuite/itemize.test index 553d0cef..bbdfc942 100644 --- a/testsuite/itemize.test +++ b/testsuite/itemize.test @@ -18,7 +18,7 @@ outfile="$scratchdir/rsync.out" makepath "$fromdir/foo" makepath "$fromdir/bar/baz" cp -p "$srcdir/configure.ac" "$fromdir/foo/config1" -cp -p "$srcdir/config.h.in" "$fromdir/foo/config2" +cp -p "$srcdir/config.sub" "$fromdir/foo/config2" cp -p "$srcdir/rsync.h" "$fromdir/bar/baz/rsync" chmod 600 "$fromdir"/foo/config? "$fromdir/bar/baz/rsync" umask 0 @@ -84,7 +84,7 @@ rm "$todir/foo/sym" umask 0 ln -s ../bar/baz "$todir/foo/sym" umask 022 -cp -p "$srcdir/config.h.in" "$fromdir/foo/config2" +cp -p "$srcdir/config.sub" "$fromdir/foo/config2" chmod 600 "$fromdir/foo/config2" chmod 777 "$todir/bar/baz/rsync" diff --git a/testsuite/rsync.fns b/testsuite/rsync.fns index 65d071ca..58701fd7 100644 --- a/testsuite/rsync.fns +++ b/testsuite/rsync.fns @@ -97,7 +97,8 @@ printmsg() { } rsync_ls_lR() { - find "$@" -print | sort | sed 's/ /\\ /g' | xargs "$TOOLDIR/tls" $TLS_ARGS + find "$@" -name .git -prune -o -name auto-build-save -prune -o -print | \ + sort | sed 's/ /\\ /g' | xargs "$TOOLDIR/tls" $TLS_ARGS } get_testuid() { @@ -296,8 +297,9 @@ use chroot = no munge symlinks = no hosts allow = localhost 127.0.0.0/24 192.168.0.0/16 10.0.0.0/8 $hostname log file = $logfile -log format = %i %h [%a] %m (%u) %l %f%L transfer logging = yes +# We don't define log format here so that the test-hidden module will default +# to the internal static string (since we had a crash trying to tweak it). exclude = ? foobar.baz max verbosity = 4 $uid_setting @@ -305,16 +307,19 @@ $gid_setting [test-from] path = $fromdir + log format = %i %h [%a] %m (%u) %l %f%L read only = yes comment = r/o [test-to] path = $todir + log format = %i %h [%a] %m (%u) %l %f%L read only = no comment = r/w [test-scratch] path = $scratchdir + log format = %i %h [%a] %m (%u) %l %f%L read only = no [test-hidden] diff --git a/token.c b/token.c index 8a335886..f15637e4 100644 --- a/token.c +++ b/token.c @@ -129,8 +129,7 @@ static void add_suffix(struct suffix_tree **prior, char ltr, const char *str) if (node->letter > ltr) break; } - if (!(newnode = new(struct suffix_tree))) - out_of_memory("add_suffix"); + newnode = new(struct suffix_tree); newnode->sibling = node; newnode->child = NULL; newnode->letter = ltr; @@ -147,8 +146,7 @@ static void add_nocompress_suffixes(const char *str) char *buf, *t; const char *f = str; - if (!(buf = new_array(char, strlen(f) + 1))) - out_of_memory("add_nocompress_suffixes"); + buf = new_array(char, strlen(f) + 1); while (*f) { if (*f == '/') { @@ -186,8 +184,7 @@ static void init_set_compression(void) else f = lp_dont_compress(module_id); - if (!(match_list = t = new_array(char, strlen(f) + 2))) - out_of_memory("set_compression"); + match_list = t = new_array(char, strlen(f) + 2); per_file_default_level = do_compression_level; @@ -282,11 +279,8 @@ static int32 simple_recv_token(int f, char **data) static char *buf; int32 n; - if (!buf) { + if (!buf) buf = new_array(char, CHUNK_SIZE); - if (!buf) - out_of_memory("simple_recv_token"); - } if (residue == 0) { int32 i = read_int(f); @@ -373,8 +367,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in rprintf(FERROR, "compression init failed\n"); exit_cleanup(RERR_PROTOCOL); } - if ((obuf = new_array(char, OBUF_SIZE)) == NULL) - out_of_memory("send_deflated_token"); + obuf = new_array(char, OBUF_SIZE); init_done = 1; } else deflateReset(&tx_strm); @@ -383,8 +376,7 @@ send_deflated_token(int f, int32 token, struct map_struct *buf, OFF_T offset, in flush_pending = 0; } else if (last_token == -2) { run_start = token; - } else if (nb != 0 || token != last_token + 1 - || token >= run_start + 65536) { + } else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) { /* output previous run */ r = run_start - last_run_end; n = last_token - run_start; @@ -518,9 +510,8 @@ static int32 recv_deflated_token(int f, char **data) rprintf(FERROR, "inflate init failed\n"); exit_cleanup(RERR_PROTOCOL); } - if (!(cbuf = new_array(char, MAX_DATA_COUNT)) - || !(dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE)))) - out_of_memory("recv_deflated_token"); + cbuf = new_array(char, MAX_DATA_COUNT); + dbuf = new_array(char, AVAIL_OUT_SIZE(CHUNK_SIZE)); init_done = 1; } else { inflateReset(&rx_strm); @@ -687,7 +678,6 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of /* initialization */ if (!comp_init_done) { - zstd_cctx = ZSTD_createCCtx(); if (!zstd_cctx) { rprintf(FERROR, "compression init failed\n"); @@ -695,8 +685,6 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of } obuf = new_array(char, OBUF_SIZE); - if (!obuf) - out_of_memory("send_deflated_token"); ZSTD_CCtx_setParameter(zstd_cctx, ZSTD_c_compressionLevel, do_compression_level); zstd_out_buff.dst = obuf + 2; @@ -710,10 +698,7 @@ static void send_zstd_token(int f, int32 token, struct map_struct *buf, OFF_T of flush_pending = 0; } else if (last_token == -2) { run_start = token; - - } else if (nb != 0 || token != last_token + 1 - || token >= run_start + 65536) { - + } else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) { /* output previous run */ r = run_start - last_run_end; n = last_token - run_start; @@ -795,7 +780,6 @@ static int32 recv_zstd_token(int f, char **data) int r; if (!decomp_init_done) { - zstd_dctx = ZSTD_createDCtx(); if (!zstd_dctx) { rprintf(FERROR, "ZSTD_createDStream failed\n"); @@ -806,8 +790,6 @@ static int32 recv_zstd_token(int f, char **data) out_buffer_size = ZSTD_DStreamOutSize() * 2; cbuf = new_array(char, MAX_DATA_COUNT); dbuf = new_array(char, out_buffer_size); - if (!cbuf || !dbuf) - out_of_memory("recv_zstd_token"); zstd_in_buff.src = cbuf; zstd_out_buff.dst = dbuf; @@ -815,30 +797,31 @@ static int32 recv_zstd_token(int f, char **data) decomp_init_done = 1; } - do { - switch (recv_state) { - case r_init: - recv_state = r_idle; - rx_token = 0; - break; + for (;;) { + switch (recv_state) { + case r_init: + recv_state = r_idle; + rx_token = 0; + break; - case r_idle: - flag = read_byte(f); - if ((flag & 0xC0) == DEFLATED_DATA) { - n = ((flag & 0x3f) << 8) + read_byte(f); - read_buf(f, cbuf, n); + case r_idle: + flag = read_byte(f); + if ((flag & 0xC0) == DEFLATED_DATA) { + n = ((flag & 0x3f) << 8) + read_byte(f); + read_buf(f, cbuf, n); - zstd_in_buff.size = n; - zstd_in_buff.pos = 0; + zstd_in_buff.size = n; + zstd_in_buff.pos = 0; - recv_state = r_inflating; + recv_state = r_inflating; + break; + } - } else if (flag == END_FLAG) { - /* that's all folks */ - recv_state = r_init; - return 0; - - } else { + if (flag == END_FLAG) { + /* that's all folks */ + recv_state = r_init; + return 0; + } /* here we have a token of some kind */ if (flag & TOKEN_REL) { rx_token += flag & 0x3f; @@ -851,45 +834,42 @@ static int32 recv_zstd_token(int f, char **data) recv_state = r_running; } return -1 - rx_token; - } - break; - case r_inflating: - zstd_out_buff.size = out_buffer_size; - zstd_out_buff.pos = 0; + case r_inflated: /* zstd doesn't get into this state */ + break; - r = ZSTD_decompressStream(zstd_dctx, &zstd_out_buff, &zstd_in_buff); - n = zstd_out_buff.pos; - if (ZSTD_isError(r)) { - rprintf(FERROR, "ZSTD decomp returned %d (%d bytes)\n", r, n); - exit_cleanup(RERR_STREAMIO); - } + case r_inflating: + zstd_out_buff.size = out_buffer_size; + zstd_out_buff.pos = 0; - /* - * If the input buffer is fully consumed and the output - * buffer is not full then next step is to read more - * data. - */ - if (zstd_in_buff.size == zstd_in_buff.pos && n < out_buffer_size) - recv_state = r_idle; + r = ZSTD_decompressStream(zstd_dctx, &zstd_out_buff, &zstd_in_buff); + n = zstd_out_buff.pos; + if (ZSTD_isError(r)) { + rprintf(FERROR, "ZSTD decomp returned %d (%d bytes)\n", r, n); + exit_cleanup(RERR_STREAMIO); + } - if (n != 0) { - *data = dbuf; - return n; - } - break; + /* + * If the input buffer is fully consumed and the output + * buffer is not full then next step is to read more + * data. + */ + if (zstd_in_buff.size == zstd_in_buff.pos && n < out_buffer_size) + recv_state = r_idle; - case r_running: - ++rx_token; - if (--rx_run == 0) - recv_state = r_idle; - return -1 - rx_token; - break; + if (n != 0) { + *data = dbuf; + return n; + } + break; - case r_inflated: - break; + case r_running: + ++rx_token; + if (--rx_run == 0) + recv_state = r_idle; + return -1 - rx_token; + } } - } while (1); } #endif /* SUPPORT_ZSTD */ @@ -903,8 +883,7 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, if (last_token == -1) { if (!init_done) { - if ((obuf = new_array(char, size)) == NULL) - out_of_memory("send_compressed_token"); + obuf = new_array(char, size); init_done = 1; } last_run_end = 0; @@ -912,8 +891,7 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, flush_pending = 0; } else if (last_token == -2) { run_start = token; - } else if (nb != 0 || token != last_token + 1 - || token >= run_start + 65536) { + } else if (nb != 0 || token != last_token + 1 || token >= run_start + 65536) { /* output previous run */ r = run_start - last_run_end; n = last_token - run_start; @@ -938,7 +916,6 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, const char *next_in; do { - char *ptr = obuf; char *next_out = obuf + 2; if (available_out == 0) { @@ -953,10 +930,10 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, exit_cleanup(RERR_STREAMIO); } if (available_out <= MAX_DATA_COUNT) { - ptr[0] = DEFLATED_DATA + (available_out >> 8); - ptr[1] = available_out; + obuf[0] = DEFLATED_DATA + (available_out >> 8); + obuf[1] = available_out; - write_buf(f, ptr, available_out + 2); + write_buf(f, obuf, available_out + 2); available_out = 0; nb -= available_in; @@ -965,14 +942,14 @@ send_compressed_token(int f, int32 token, struct map_struct *buf, OFF_T offset, } while (nb != 0); flush_pending = token == -2; } - if (token == -1) + if (token == -1) { /* end of file - clean up */ write_byte(f, END_FLAG); + } } static int32 recv_compressed_token(int f, char **data) { - static int32 saved_flag; static int init_done; int32 n, flag; int size = MAX(LZ4_compressBound(CHUNK_SIZE), MAX_DATA_COUNT+2); @@ -984,21 +961,16 @@ static int32 recv_compressed_token(int f, char **data) switch (recv_state) { case r_init: if (!init_done) { - if (!(cbuf = new_array(char, MAX_DATA_COUNT)) - || !(dbuf = new_array(char, size))) - out_of_memory("recv_compressed_token"); + cbuf = new_array(char, MAX_DATA_COUNT); + dbuf = new_array(char, size); init_done = 1; } recv_state = r_idle; rx_token = 0; break; + case r_idle: - case r_inflated: - if (saved_flag) { - flag = saved_flag & 0xff; - saved_flag = 0; - } else - flag = read_byte(f); + flag = read_byte(f); if ((flag & 0xC0) == DEFLATED_DATA) { n = ((flag & 0x3f) << 8) + read_byte(f); read_buf(f, cbuf, n); @@ -1008,9 +980,6 @@ static int32 recv_compressed_token(int f, char **data) break; } - if (recv_state == r_inflated) - recv_state = r_idle; - if (flag == END_FLAG) { /* that's all folks */ recv_state = r_init; @@ -1036,10 +1005,13 @@ static int32 recv_compressed_token(int f, char **data) rprintf(FERROR, "uncompress failed: %d\n", avail_out); exit_cleanup(RERR_STREAMIO); } - recv_state = r_inflated; + recv_state = r_idle; *data = dbuf; return avail_out; + case r_inflated: /* lz4 doesn't get into this state */ + break; + case r_running: ++rx_token; if (--rx_run == 0) @@ -1047,53 +1019,7 @@ static int32 recv_compressed_token(int f, char **data) return -1 - rx_token; } } - } - -# if 0 -static void see_uncompressed_token(char *buf, int32 len) -{ - static const char *next_in; - static int avail_in; - int avail_out; - - int32 blklen; - char hdr[5]; - - avail_in = 0; - blklen = 0; - hdr[0] = 0; - do { - if (avail_in == 0 && len != 0) { - if (blklen == 0) { - /* Give it a fake stored-block header. */ - next_in = hdr; - avail_in = 5; - blklen = len; - if (blklen > 0xffff) - blklen = 0xffff; - hdr[1] = blklen; - hdr[2] = blklen >> 8; - hdr[3] = ~hdr[1]; - hdr[4] = ~hdr[2]; - } else { - next_in = (char *)buf; - avail_in = blklen; - if (protocol_version >= 31) /* Newer protocols avoid a data-duplicating bug */ - buf += blklen; - len -= blklen; - blklen = 0; - } - } - avail_out = LZ4_decompress_safe(next_in, dbuf, avail_in, LZ4_compressBound(CHUNK_SIZE)); - if (avail_out < 0) { - rprintf(FERROR, "uncompress failed: %d\n", avail_out); - exit_cleanup(RERR_STREAMIO); - } - - } while (len); -} -# endif /* 0 */ #endif /* SUPPORT_LZ4 */ /** diff --git a/uidlist.c b/uidlist.c index 739286a4..7359d9db 100644 --- a/uidlist.c +++ b/uidlist.c @@ -85,8 +85,6 @@ static struct idlist *add_to_list(struct idlist **root, id_t id, union name_or_i id_t id2, uint16 flags) { struct idlist *node = new(struct idlist); - if (!node) - out_of_memory("add_to_list"); node->next = *root; node->u = noiu; node->id = id; @@ -160,8 +158,6 @@ static int is_in_group(gid_t gid) if ((ngroups = getgroups(0, NULL)) < 0) ngroups = 0; gidset = new_array(GETGROUPS_T, ngroups+1); - if (!gidset) - out_of_memory("is_in_group"); if (ngroups > 0) ngroups = getgroups(ngroups, gidset); /* The default gid might not be in the list on some systems. */ @@ -174,8 +170,6 @@ static int is_in_group(gid_t gid) if (DEBUG_GTE(OWN, 2)) { int pos; char *gidbuf = new_array(char, ngroups*21+32); - if (!gidbuf) - out_of_memory("is_in_group"); pos = snprintf(gidbuf, 32, "process has %d gid%s: ", ngroups, ngroups == 1? "" : "s"); for (n = 0; n < ngroups; n++) { pos += snprintf(gidbuf+pos, 21, " %d", (int)gidset[n]); @@ -375,8 +369,6 @@ uid_t recv_user_name(int f, uid_t uid) struct idlist *node; int len = read_byte(f); char *name = new_array(char, len+1); - if (!name) - out_of_memory("recv_user_name"); read_sbuf(f, name, len); if (numeric_ids < 0) { free(name); @@ -391,8 +383,6 @@ gid_t recv_group_name(int f, gid_t gid, uint16 *flags_ptr) struct idlist *node; int len = read_byte(f); char *name = new_array(char, len+1); - if (!name) - out_of_memory("recv_group_name"); read_sbuf(f, name, len); if (numeric_ids < 0) { free(name); diff --git a/util.c b/util.c index 84dcac52..3a50e44f 100644 --- a/util.c +++ b/util.c @@ -592,8 +592,7 @@ int lock_range(int fd, int offset, int len) } #define ENSURE_MEMSPACE(buf, type, sz, req) \ - if ((req) > sz && !(buf = realloc_array(buf, type, sz = MAX(sz * 2, req)))) \ - out_of_memory("glob_expand") + do { if ((req) > sz) buf = realloc_array(buf, type, sz = MAX(sz * 2, req)); } while(0) static inline void call_glob_match(const char *name, int len, int from_glob, char *arg, int abpos, int fbpos); @@ -695,8 +694,7 @@ static inline void call_glob_match(const char *name, int len, int from_glob, glob_match(arg, abpos, fbpos); } else { ENSURE_MEMSPACE(glob.argv, char *, glob.maxargs, glob.argc + 1); - if (!(glob.argv[glob.argc++] = strdup(glob.arg_buf))) - out_of_memory("glob_match"); + glob.argv[glob.argc++] = strdup(glob.arg_buf); } } @@ -720,8 +718,6 @@ int glob_expand(const char *arg, char ***argv_p, int *argc_p, int *maxargs_p) s = sanitize_path(NULL, arg, "", 0, SP_KEEP_DOT_DIRS); else { s = strdup(arg); - if (!s) - out_of_memory("glob_expand"); clean_fname(s, CFN_KEEP_DOT_DIRS | CFN_KEEP_TRAILING_SLASH | CFN_COLLAPSE_DOT_DOT_DIRS); } @@ -771,8 +767,7 @@ void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int return; } - if (!(arg = strdup(arg))) - out_of_memory("glob_expand_module"); + arg = strdup(arg); if (asprintf(&base," %s/", base1) < 0) out_of_memory("glob_expand_module"); @@ -1017,11 +1012,10 @@ char *sanitize_path(char *dest, const char *p, const char *rootdir, int depth, i depth = 0; p++; } - if (dest) { - if (rlen + plen + 1 >= MAXPATHLEN) - return NULL; - } else if (!(dest = new_array(char, MAX(rlen + plen + 1, 2)))) - out_of_memory("sanitize_path"); + if (!dest) + dest = new_array(char, MAX(rlen + plen + 1, 2)); + else if (rlen + plen + 1 >= MAXPATHLEN) + return NULL; if (rlen) { /* only true if p previously started with a slash */ memcpy(dest, rootdir, rlen); if (rlen > 1) /* a rootdir of len 1 is "/", so this avoids a 2nd slash */ @@ -1155,13 +1149,10 @@ char *normalize_path(char *path, BOOL force_newbuf, unsigned int *len_ptr) return NULL; curr_dir[curr_dir_len] = '/'; memcpy(curr_dir + curr_dir_len + 1, path, len + 1); - if (!(path = strdup(curr_dir))) - out_of_memory("normalize_path"); + path = strdup(curr_dir); curr_dir[curr_dir_len] = '\0'; - } else if (force_newbuf) { - if (!(path = strdup(path))) - out_of_memory("normalize_path"); - } + } else if (force_newbuf) + path = strdup(path); len = clean_fname(path, CFN_COLLAPSE_DOT_DOT_DIRS | CFN_DROP_TRAILING_DOT_DIR); @@ -1519,8 +1510,7 @@ struct bitbag *bitbag_create(int max_ndx) struct bitbag *bb = new(struct bitbag); bb->slot_cnt = (max_ndx + BB_PER_SLOT_BITS - 1) / BB_PER_SLOT_BITS; - if (!(bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*)))) - out_of_memory("bitbag_create"); + bb->bits = (uint32**)calloc(bb->slot_cnt, sizeof (uint32*)); return bb; } @@ -1601,8 +1591,7 @@ void flist_ndx_push(flist_ndx_list *lp, int ndx) { struct flist_ndx_item *item; - if (!(item = new(struct flist_ndx_item))) - out_of_memory("flist_ndx_push"); + item = new(struct flist_ndx_item); item->next = NULL; item->ndx = ndx; if (lp->tail) @@ -1654,8 +1643,7 @@ void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int in new_size = 1; if (new_size <= lp->malloced) overflow_exit("expand_item_list"); - /* Using _realloc_array() lets us pass the size, not a type. */ - new_ptr = _realloc_array(lp->items, item_size, new_size); + new_ptr = realloc_buf(lp->items, new_size * item_size); if (DEBUG_GTE(FLIST, 3)) { rprintf(FINFO, "[%s] expand %s to %s bytes, did%s move\n", who_am_i(), desc, big_num(new_size * item_size), diff --git a/util2.c b/util2.c index ad7b0636..181dbd7d 100644 --- a/util2.c +++ b/util2.c @@ -25,6 +25,10 @@ #include "itypes.h" #include "inums.h" +extern size_t max_alloc; + +char *do_malloc = "42"; + /** * Sleep for a specified number of milliseconds. * @@ -67,22 +71,39 @@ int msleep(int t) return True; } -#define MALLOC_MAX 0x40000000 - -void *_new_array(unsigned long num, unsigned int size, int use_calloc) +/* Convert a num manually because the needed %lld precision is not a portable sprintf() escape. */ +char *num_to_byte_string(ssize_t num) { - if (num >= MALLOC_MAX/size) - return NULL; - return use_calloc ? calloc(num, size) : malloc(num * size); + char buf[128], *s = buf + sizeof buf - 1; + + *s = '\0'; + while (num) { + *--s = (char)(num % 10) + '0'; + num /= 10; + } + return strdup(s); } -void *_realloc_array(void *ptr, unsigned int size, size_t num) +void *my_alloc(void *ptr, size_t num, size_t size, const char *file, int line) { - if (num >= MALLOC_MAX/size) - return NULL; + if (num >= max_alloc/size) { + if (!file) + return NULL; + rprintf(FERROR, "[%s] exceeded --max-alloc=%s setting (file=%s, line=%d)\n", + who_am_i(), num_to_byte_string(max_alloc), file, line); + exit_cleanup(RERR_MALLOC); + } if (!ptr) - return malloc(size * num); - return realloc(ptr, size * num); + ptr = calloc(num, size); + else if (ptr == do_malloc) + ptr = malloc(num * size); + else + ptr = realloc(ptr, num * size); + if (!ptr && file) { + rprintf(FERROR, "[%s] out of memory (file=%s, line=%d)\n", who_am_i(), file, line); + exit_cleanup(RERR_MALLOC); + } + return ptr; } const char *sum_as_hex(int csum_type, const char *sum, int flist_csum) diff --git a/version.h b/version.h new file mode 100644 index 00000000..34fbda49 --- /dev/null +++ b/version.h @@ -0,0 +1 @@ +#define RSYNC_VERSION "3.2.2" diff --git a/xattrs.c b/xattrs.c index bbe003b9..b3f0c1a3 100644 --- a/xattrs.c +++ b/xattrs.c @@ -145,8 +145,6 @@ static ssize_t get_xattr_names(const char *fname) if (!namebuf) { namebuf_len = 1024; namebuf = new_array(char, namebuf_len); - if (!namebuf) - out_of_memory("get_xattr_names"); } while (1) { @@ -174,8 +172,6 @@ static ssize_t get_xattr_names(const char *fname) free(namebuf); namebuf_len = list_len + 1024; namebuf = new_array(char, namebuf_len); - if (!namebuf) - out_of_memory("get_xattr_names"); } return list_len; @@ -205,8 +201,7 @@ static char *get_xattr_data(const char *fname, const char *name, size_t *len_ptr extra_len = 1; /* request non-zero amount of memory */ if (datum_len + extra_len < datum_len) overflow_exit("get_xattr_data"); - if (!(ptr = new_array(char, datum_len + extra_len))) - out_of_memory("get_xattr_data"); + ptr = new_array(char, datum_len + extra_len); if (datum_len) { size_t len = sys_lgetxattr(fname, name, ptr, datum_len); @@ -279,8 +274,7 @@ static int rsync_xal_get(const char *fname, item_list *xalp) sum_update(ptr, datum_len); free(ptr); - if (!(ptr = new_array(char, name_offset + name_len))) - out_of_memory("rsync_xal_get"); + ptr = new_array(char, name_offset + name_len); *ptr = XSTATE_ABBREV; sum_end(ptr + 1); } else @@ -481,8 +475,6 @@ static int rsync_xal_store(item_list *xalp) out_of_memory("rsync_xal_h hashtable_create()"); new_ref = new0(rsync_xa_list_ref); - if (new_ref == NULL) - out_of_memory("new0(rsync_xa_list_ref)"); new_ref->ndx = ndx; node = hashtable_find(rsync_xal_h, new_list->key, new_ref); @@ -759,8 +751,6 @@ int recv_xattr_request(struct file_struct *file, int f_in) if (rxa->name_len + rxa->datum_len < rxa->name_len) overflow_exit("recv_xattr_request"); rxa->datum = new_array(char, rxa->datum_len + rxa->name_len); - if (!rxa->datum) - out_of_memory("recv_xattr_request"); name = rxa->datum + rxa->datum_len; memcpy(name, rxa->name, rxa->name_len); rxa->name = name; @@ -813,8 +803,6 @@ void receive_xattr(int f, struct file_struct *file) || (dget_len + extra_len + name_len < dget_len + extra_len)) overflow_exit("receive_xattr"); ptr = new_array(char, dget_len + extra_len + name_len); - if (!ptr) - out_of_memory("receive_xattr"); name = ptr + dget_len + extra_len; read_buf(f, name, name_len); if (name_len < 1 || name[name_len-1] != '\0') { diff -upN a/config.h.in b/config.h.in --- a/config.h.in +++ b/config.h.in @@ -697,9 +697,6 @@ /* Define to 1 if --protected-args should be the default */ #undef RSYNC_USE_PROTECTED_ARGS -/* rsync release version */ -#undef RSYNC_VERSION - /* Define to 1 if sockets need to be shutdown */ #undef SHUTDOWN_ALL_SOCKETS diff -upN a/configure.sh b/configure.sh --- a/configure.sh +++ b/configure.sh @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for rsync 3.2.1. +# Generated by GNU Autoconf 2.69 for rsync . # # Report bugs to . # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='rsync' PACKAGE_TARNAME='rsync' -PACKAGE_VERSION='3.2.1' -PACKAGE_STRING='rsync 3.2.1' +PACKAGE_VERSION=' ' +PACKAGE_STRING='rsync ' PACKAGE_BUGREPORT='https://rsync.samba.org/bugtracking.html' PACKAGE_URL='' @@ -667,7 +667,6 @@ build_os build_vendor build_cpu build -RSYNC_VERSION target_alias host_alias build_alias @@ -1297,7 +1296,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures rsync 3.2.1 to adapt to many kinds of systems. +\`configure' configures rsync to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1363,7 +1362,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of rsync 3.2.1:";; + short | recursive ) echo "Configuration of rsync :";; esac cat <<\_ACEOF @@ -1482,7 +1481,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -rsync configure 3.2.1 +rsync configure generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2225,7 +2224,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by rsync $as_me 3.2.1, which was +It was created by rsync $as_me , which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -2581,17 +2580,11 @@ ac_config_headers="$ac_config_headers co -RSYNC_VERSION=$PACKAGE_VERSION +PACKAGE_VERSION=`sed 's/.*"\(.*\)".*/\1/' <$srcdir/version.h` { $as_echo "$as_me:${as_lineno-$LINENO}: Configuring rsync $PACKAGE_VERSION" >&5 $as_echo "$as_me: Configuring rsync $PACKAGE_VERSION" >&6;} - -cat >>confdefs.h <<_ACEOF -#define RSYNC_VERSION "$PACKAGE_VERSION" -_ACEOF - - LDFLAGS=${LDFLAGS-""} ac_aux_dir= @@ -10558,7 +10551,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_wri # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by rsync $as_me 3.2.1, which was +This file was extended by rsync $as_me , which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -10620,7 +10613,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -rsync config.status 3.2.1 +rsync config.status configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -11345,7 +11338,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: " >&5 $as_echo "" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: rsync ${RSYNC_VERSION} configuration successful" >&5 -$as_echo " rsync ${RSYNC_VERSION} configuration successful" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: rsync $PACKAGE_VERSION configuration successful" >&5 +$as_echo " rsync $PACKAGE_VERSION configuration successful" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: " >&5 $as_echo "" >&6; } diff -upN a/proto.h b/proto.h --- a/proto.h +++ b/proto.h @@ -54,6 +54,7 @@ void set_allow_inc_recurse(void); void parse_compress_choice(int final_call); struct name_num_item *get_nni_by_name(struct name_num_obj *nno, const char *name, int len); struct name_num_item *get_nni_by_num(struct name_num_obj *nno, int num); +void validate_choice_vs_env(int ntype, int num1, int num2); int get_default_nno_list(struct name_num_obj *nno, char *to_buf, int to_buf_len, char dup_markup); void setup_protocol(int f_out,int f_in); int claim_connection(char *fname, int max_connections); @@ -186,60 +187,6 @@ int io_end_multiplex_out(int mode); void start_write_batch(int fd); void stop_write_batch(void); void reset_daemon_vars(void); -char *lp_bind_address(void); -char *lp_daemon_chroot(void); -char *lp_daemon_gid(void); -char *lp_daemon_uid(void); -char *lp_motd_file(void); -char *lp_pid_file(void); -char *lp_socket_options(void); -int lp_listen_backlog(void); -int lp_rsync_port(void); -BOOL lp_proxy_protocol(void); -char *lp_auth_users(int module_id); -char *lp_charset(int module_id); -char *lp_comment(int module_id); -char *lp_dont_compress(int module_id); -char *lp_early_exec(int module_id); -char *lp_exclude(int module_id); -char *lp_exclude_from(int module_id); -char *lp_filter(int module_id); -char *lp_gid(int module_id); -char *lp_hosts_allow(int module_id); -char *lp_hosts_deny(int module_id); -char *lp_include(int module_id); -char *lp_include_from(int module_id); -char *lp_incoming_chmod(int module_id); -char *lp_lock_file(int module_id); -char *lp_log_file(int module_id); -char *lp_log_format(int module_id); -char *lp_name(int module_id); -char *lp_outgoing_chmod(int module_id); -char *lp_path(int module_id); -char *lp_postxfer_exec(int module_id); -char *lp_prexfer_exec(int module_id); -char *lp_refuse_options(int module_id); -char *lp_secrets_file(int module_id); -char *lp_syslog_tag(int module_id); -char *lp_temp_dir(int module_id); -char *lp_uid(int module_id); -int lp_max_connections(int module_id); -int lp_max_verbosity(int module_id); -int lp_syslog_facility(int module_id); -int lp_timeout(int module_id); -BOOL lp_fake_super(int module_id); -BOOL lp_forward_lookup(int module_id); -BOOL lp_ignore_errors(int module_id); -BOOL lp_ignore_nonreadable(int module_id); -BOOL lp_list(int module_id); -BOOL lp_munge_symlinks(int module_id); -BOOL lp_numeric_ids(int module_id); -BOOL lp_read_only(int module_id); -BOOL lp_reverse_lookup(int module_id); -BOOL lp_strict_modes(int module_id); -BOOL lp_transfer_logging(int module_id); -BOOL lp_use_chroot(int module_id); -BOOL lp_write_only(int module_id); int lp_load(char *pszFname, int globals_only); BOOL set_dparams(int syntax_check_only); int lp_num_modules(void); @@ -407,8 +354,8 @@ int flist_ndx_pop(flist_ndx_list *lp); void *expand_item_list(item_list *lp, size_t item_size, const char *desc, int incr); void force_memzero(void *buf, size_t len); int msleep(int t); -void *_new_array(unsigned long num, unsigned int size, int use_calloc); -void *_realloc_array(void *ptr, unsigned int size, size_t num); +char *num_to_byte_string(ssize_t num); +void *my_alloc(void *ptr, size_t num, size_t size, const char *file, int line); const char *sum_as_hex(int csum_type, const char *sum, int flist_csum); NORETURN void out_of_memory(const char *str); NORETURN void overflow_exit(const char *str); @@ -434,3 +381,58 @@ int x_fstat(int fd, STRUCT_STAT *fst, ST int sys_gettimeofday(struct timeval *tv); char *do_big_num(int64 num, int human_flag, const char *fract); char *do_big_dnum(double dnum, int human_flag, int decimal_digits); +char *lp_bind_address(void); +char *lp_daemon_chroot(void); +char *lp_daemon_gid(void); +char *lp_daemon_uid(void); +char *lp_motd_file(void); +char *lp_pid_file(void); +char *lp_socket_options(void); +int lp_listen_backlog(void); +int lp_rsync_port(void); +BOOL lp_proxy_protocol(void); +char *lp_auth_users(int module_id); +char *lp_charset(int module_id); +char *lp_comment(int module_id); +char *lp_dont_compress(int module_id); +char *lp_early_exec(int module_id); +char *lp_exclude(int module_id); +char *lp_exclude_from(int module_id); +char *lp_filter(int module_id); +char *lp_gid(int module_id); +char *lp_hosts_allow(int module_id); +char *lp_hosts_deny(int module_id); +char *lp_include(int module_id); +char *lp_include_from(int module_id); +char *lp_incoming_chmod(int module_id); +char *lp_lock_file(int module_id); +char *lp_log_file(int module_id); +char *lp_log_format(int module_id); +char *lp_name(int module_id); +char *lp_outgoing_chmod(int module_id); +char *lp_postxfer_exec(int module_id); +char *lp_prexfer_exec(int module_id); +char *lp_refuse_options(int module_id); +char *lp_secrets_file(int module_id); +char *lp_syslog_tag(int module_id); +char *lp_uid(int module_id); +char *lp_path(int module_id); +char *lp_temp_dir(int module_id); +int lp_max_connections(int module_id); +int lp_max_verbosity(int module_id); +int lp_timeout(int module_id); +int lp_syslog_facility(int module_id); +BOOL lp_fake_super(int module_id); +BOOL lp_forward_lookup(int module_id); +BOOL lp_ignore_errors(int module_id); +BOOL lp_ignore_nonreadable(int module_id); +BOOL lp_list(int module_id); +BOOL lp_read_only(int module_id); +BOOL lp_reverse_lookup(int module_id); +BOOL lp_strict_modes(int module_id); +BOOL lp_transfer_logging(int module_id); +BOOL lp_use_chroot(int module_id); +BOOL lp_write_only(int module_id); +BOOL lp_munge_symlinks(int module_id); +BOOL lp_numeric_ids(int module_id); +BOOL lp_open_noatime(int module_id); diff -upN a/rsync-ssl.1 b/rsync-ssl.1 --- a/rsync-ssl.1 +++ b/rsync-ssl.1 @@ -1,4 +1,4 @@ -.TH "rsync-ssl" "1" "22 Jun 2020" "rsync-ssl 3.2.1" "User Commands" +.TH "rsync-ssl" "1" "04 Jul 2020" "rsync-ssl 3.2.2" "User Commands" .P .SH "NAME" .P @@ -116,7 +116,7 @@ Please report bugs! See the web site at .P .SH "VERSION" .P -This man page is current for version 3.2.1 of rsync. +This man page is current for version 3.2.2 of rsync. .P .SH "CREDITS" .P diff -upN a/rsync-ssl.1.html b/rsync-ssl.1.html --- a/rsync-ssl.1.html +++ b/rsync-ssl.1.html @@ -106,7 +106,7 @@ exported RSYNC_SSL_TYPE environment vari

BUGS

Please report bugs! See the web site at https://rsync.samba.org/.

VERSION

-

This man page is current for version 3.2.1 of rsync.

+

This man page is current for version 3.2.2 of rsync.

CREDITS

rsync is distributed under the GNU General Public License. See the file COPYING for details.

@@ -116,5 +116,5 @@ FAQ-O-Matic which may cover questions un

This manpage was written by Wayne Davison.

Mailing lists for support and development are available at https://lists.samba.org/.

-

22 Jun 2020

+

04 Jul 2020

diff -upN a/rsync.1 b/rsync.1 --- a/rsync.1 +++ b/rsync.1 @@ -1,4 +1,4 @@ -.TH "rsync" "1" "22 Jun 2020" "rsync 3.2.1" "User Commands" +.TH "rsync" "1" "04 Jul 2020" "rsync 3.2.2" "User Commands" .P .SH "NAME" .P @@ -479,6 +479,7 @@ detailed description below for a complet --max-delete=NUM don't delete more than NUM files --max-size=SIZE don't transfer any file larger than SIZE --min-size=SIZE don't transfer any file smaller than SIZE +--max-alloc=SIZE change a limit relating to memory alloc --partial keep partially transferred files --partial-dir=DIR put a partially transferred file into DIR --delay-updates put all updated files into place at end @@ -588,9 +589,6 @@ Print the rsync version plus other info The output includes the default list of checksum algorithms, the default list of compression algorithms, a list of compiled-in capabilities, a link to the rsync web site, and some license/copyright info. -.IP -Repeat the option (\fB\-VV\fP) to include some optimization info at the end of -the capabilities list. .IP "\fB\-\-verbose\fP, \fB\-v\fP" This option increases the amount of information you are given during the transfer. By default, rsync works silently. A single \fB\-v\fP will give you @@ -678,7 +676,7 @@ file's data is fully sent. It is also h helps to avoid overpopulating the protocol data with extra message data. .IP The option does not affect the remote side of a transfer without using -\fB\-\-remote-option\fP\ \-\- e.g. \fB\-M\-\-msgs2stderr\fP or \fB{\-M,}\-\-msgs2stderr\fP. +\fB\-\-remote-option\fP, e.g. \fB\-M\-\-msgs2stderr\fP or \fB{\-M,}\-\-msgs2stderr\fP. .IP Also keep in mind that connecting to a normal (non-remote-shell) daemon does not have a stderr channel to send messages back to the client side, so @@ -755,7 +753,7 @@ after-the-transfer verification has noth before-the-transfer "Does this file need to be updated?" check. .IP The checksum used is auto-negotiated between the client and the server, but -can be overridden using either the \fB\-\-checksum-choice\fP option or an +can be overridden using either the \fB\-\-checksum-choice\fP (\fB\-\-cc\fP) option or an environment variable that is discussed in that option's section. .IP "\fB\-\-archive\fP, \fB\-a\fP" This is equivalent to \fB\-rlptgoD\fP. It is a quick way of saying you want @@ -1527,9 +1525,13 @@ The checksum options that you may be abl .IP .RS .IP o -\fBauto\fP (the default) +\fBauto\fP (the default automatic choice) +.IP o +\fBxxh128\fP .IP o -\fBxxh64\fP (aka xxhash) +\fBxxh3\fP +.IP o +\fBxxh64\fP (aka \fBxxhash\fP) .IP o \fBmd5\fP .IP o @@ -1538,7 +1540,8 @@ The checksum options that you may be abl \fBnone\fP .RE .IP -Run \fBrsync\ \-V\fP to see the default checksum list compiled into your version. +Run \fBrsync\ \-\-version\fP to see the default checksum list compiled into your +version (which may differ from the list above). .IP If "none" is specified for the first (or only) name, the \fB\-\-whole-file\fP option is forced on and no checksum verification is performed on the @@ -1548,22 +1551,22 @@ the \fB\-\-checksum\fP option cannot be The "auto" option is the default, where rsync bases its algorithm choice on a negotiation between the client and the server as follows: .IP -If both the client and the server are at least version 3.2.0, they will -exchange a list of checksum names and choose the first one in the list that -they have in common. This typically means that they will choose xxh64 if -they both support it and fall back to MD5. If one side of the transfer is -not new enough to support this checksum negotiation, then a value is chosen -based on the protocol version (which chooses between MD5 and various -flavors of MD4 based on protocol age). -.IP -You can also override the checksum using the RSYNC_CHECKSUM_LIST -environment variable by setting it to a space-separated list of checksum -names that you consider acceptable. If no common checksum is found, the -client exits with an error. This method does not allow you to specify the -transfer checksum separately from the pre-transfer checksum, and it ignores -"auto" and all unknown checksum names. If the remote rsync is not new -enough to handle a checksum negotiation list, the list is silently ignored -unless it contains the string "FAIL". +When both sides of the transfer are at least 3.2.0, rsync chooses the first +algorithm in the client's list of choices that is also in the server's list +of choices. If no common checksum choice is found, rsync exits with +an error. If the remote rsync is too old to support checksum negotiation, +a value is chosen based on the protocol version (which chooses between MD5 +and various flavors of MD4 based on protocol age). +.IP +The default order can be customized by setting the environment variable +RSYNC_CHECKSUM_LIST to a space-separated list of acceptable checksum names. +If the string contains a "\fB&\fP" character, it is separated into the "client +string & server string", otherwise the same string +applies to both. If the string (or string portion) contains no +non-whitespace characters, the default checksum list is used. This method +does not allow you to specify the transfer checksum separately from the +pre-transfer checksum, and it discards "auto" and all unknown checksum +names. A list with only invalid names results in a failed negotiation. .IP The use of the \fB\-\-checksum-choice\fP option overrides this environment list. .IP "\fB\-\-one-file-system\fP, \fB\-x\fP" @@ -1758,12 +1761,16 @@ This option is a transfer rule, not an e data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred. .IP -The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or -"MiB") is a mebibyte (1024*1024), and "G" (or "GiB") is a gibibyte -(1024*1024*1024). If you want the multiplier to be 1000 instead of 1024, -use "KB", "MB", or "GB". (Note: lower-case is also accepted for all -values.) Finally, if the suffix ends in either "+1" or "\-1", the value will -be offset by one byte in the indicated direction. +The accepted suffix letters are: \fBB\fP, \fBK\fP, \fBG\fP, \fBT\fP, and \fBP\fP for bytes, +kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes, +terabytes/tebibytes, and petabytes/pebibytes. If you use a single-char +suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are +multiples of 1024. If you use a two-letter suffix that ends with a "B" +(e.g. "kb") then you get units that are multiples of 1000. The suffix +letters can be any mix of upper and lower-case that you want to use. +.IP +Finally, if the string ends with either "+1" or "\-1", it is offset by one +byte in the indicated direction. The largest possible value is \fB8192P-1\fP. .IP Examples: \fB\-\-max-size=1.5mb-1\fP is 1499999 bytes, and \fB\-\-max-size=2g+1\fP is 2147483649 bytes. @@ -1775,6 +1782,26 @@ specified SIZE, which can help in not tr the \fB\-\-max-size\fP option for a description of SIZE and other information. .IP Note that rsync versions prior to 3.1.0 did not allow \fB\-\-min-size=0\fP. +.IP "\fB\-\-max-alloc=SIZE\fP" +By default rsync limits an individual malloc/realloc to about 1GB in size. +For most people this limit works just fine and prevents a protocol error +causing rsync to request massive amounts of memory. However, if you have +many millions of files in a transfer, a large amount of server memory, and +you don't want to split up your transfer into multiple parts, you can +increase the per-allocation limit to something larger and rsync will +consume more memory. +.IP +Keep in mind that this is not a limit on the total size of allocated +memory. It is a sanity-check value for each individual allocation. +.IP +See the \fB\-\-max-size\fP option for a description of how SIZE can be specified. +The default suffix if none is given is bytes. +.IP +You can set a default value using the environment variable RSYNC_MAX_ALLOC +using the same SIZE values as supported by this option. If the remote +rsync doesn't understand the \fB\-\-max-alloc\fP option, you can override an +environmental value by specifying \fB\-\-max-alloc=1g\fP, which will make rsync +avoid sending the option to the remote side (because "1G" is the default). .IP "\fB\-\-block-size=SIZE\fP, \fB\-B\fP" This forces the block size used in rsync's delta-transfer algorithm to a fixed value. It is normally selected based on the size of each file being @@ -2108,8 +2135,9 @@ that is older than that. .IP Rsync can also be configured (at build time) to have this option enabled by default (with is overridden by both the environment and the command-line). -Run \fBrsync\ \-V\fP to check if this is the case, as it will display "default -protect-args" or "optional protect-args" depending on how it was compiled. +Run \fBrsync\ \-\-version\fP to check if this is the case, as it will display +"default protect-args" or "optional protect-args" depending on how it was +compiled. .IP This option will eventually become a new default setting at some as-yet-undetermined point in the future. @@ -2293,27 +2321,40 @@ With this option, rsync compresses the f destination machine, which reduces the amount of data being transmitted\ \-\- something that is useful over a slow connection. .IP -The "zlib" compression method typically achieves better compression ratios -than can be achieved by using a compressing remote shell or a compressing -transport because it takes advantage of the implicit information in the -matching data blocks that are not explicitly sent over the connection. -This matching-data compression comes at a cost of CPU, though, and can be -disabled by using the "zlibx" compresson method instead. This can be -selected by repeating the \fB\-z\fP option or specifying -\fB\-\-compress-choice=zlibx\fP, but it only works if both sides of the transfer -are at least version 3.1.1. +Rsync supports multiple compression methods and will choose one for you +unless you force the choice using the \fB\-\-compress-choice\fP (\fB\-\-zc\fP) option. .IP -Note that if you see an error about an option named \fB\-\-old-compress\fP or -\fB\-\-new-compress\fP, this is rsync trying to send the \fB\-\-compress-choice=zlib\fP -or \fB\-\-compress-choice=zlibx\fP option in a backward-compatible manner that -more rsync versions understand. This error indicates that the older rsync -version will not allow you to force the compression type. +Run \fBrsync\ \-\-version\fP to see the default compress list compiled into your +version. .IP -See the \fB\-\-skip-compress\fP option for the default list of file suffixes that -will not be compressed. +When both sides of the transfer are at least 3.2.0, rsync chooses the first +algorithm in the client's list of choices that is also in the server's list +of choices. If no common compress choice is found, rsync exits with +an error. If the remote rsync is too old to support checksum negotiation, +its list is assumed to be "zlib". +.IP +The default order can be customized by setting the environment variable +RSYNC_COMPRESS_LIST to a space-separated list of acceptable compression +names. If the string contains a "\fB&\fP" character, it is separated into the +"client string & server string", otherwise the same string applies to both. +If the string (or string portion) contains no +non-whitespace characters, the default compress list is used. Any unknown +compression names are discarded from the list, but a list with only invalid +names results in a failed negotiation. +.IP +There are some older rsync versions that were configured to reject a \fB\-z\fP +option and require the use of \fB\-zz\fP because their compression library was +not compatible with the default zlib compression method. You can usually +ignore this weirdness unless the rsync server complains and tells you to +specify \fB\-zz\fP. +.IP +See also the \fB\-\-skip-compress\fP option for the default list of file suffixes +that will trasnferred with no (or minimal) compression. .IP "\fB\-\-compress-choice=STR\fP, \fB\-\-zc=STR\fP" -This option can be used to override the automatic selection of the -compression algorithm that is the default when \fB\-\-compress\fP is used. +This option can be used to override the automatic negotiation of the +compression algorithm that occurs when \fB\-\-compress\fP is used. The option +implies \fB\-\-compress\fP unless "none" was specified, which instead implies +\fB\-\-no-compress\fP. .IP The compression options that you may be able to use are: .IP @@ -2330,25 +2371,18 @@ The compression options that you may be \fBnone\fP .RE .IP -Run \fBrsync\ \-V\fP to see the compress list compiled into your version. +Run \fBrsync\ \-\-version\fP to see the default compress list compiled into your +version (which may differ from the list above). +.IP +Note that if you see an error about an option named \fB\-\-old-compress\fP or +\fB\-\-new-compress\fP, this is rsync trying to send the \fB\-\-compress-choice=zlib\fP +or \fB\-\-compress-choice=zlibx\fP option in a backward-compatible manner that +more rsync versions understand. This error indicates that the older rsync +version on the server will not allow you to force the compression type. .IP -The "zlibx" algorithm is given preference over "zlib" if both sides of the -transfer are at least version 3.2.0, otherwise it will choose "zlib" unless -you override it via something like \fB\-zz\fP. These 2 algorithms are the stame -except that "zlibx" does not try to include matched data that was not -transferred in the compression computations. -.IP -If "none" is specified, that is equivalent to using \fB\-\-no-compress\fP. -.IP -This option implies \fB\-\-compress\fP unless "none" was specified. -.IP -You can also override the compression negotiation using the -RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated -list of compression names that you consider acceptable. If no common -compress choice is found, the client exits with an error. It ignores -"auto" and all unknown compression names. If the remote rsync is not new -enough to handle a compression negotiation list, the list is silently -ignored unless it contains the string "FAIL". +Note that the "zlibx" compression algorithm is just the "zlib" algorithm +with matched data excluded from the compression stream (to try to make it +more compatible with an external zlib implementation). .IP "\fB\-\-compress-level=NUM\fP, \fB\-\-zl=NUM\fP" Explicitly set the compression level to use (see \fB\-\-compress\fP, \fB\-z\fP) instead of letting it default. The \fB\-\-compress\fP option is implied as long @@ -2357,8 +2391,8 @@ algorithm that is in effect (e.g. zlib c "off"). .IP The level values vary depending on the checksum in effect. Because rsync -will negotiate a checksum choice by default when the remote rsync is new -enough, it can be good to combine this option with a \fB\-\-compress-choice\fP +will negotiate a checksum choice by default (when the remote rsync is new +enough), it can be good to combine this option with a \fB\-\-compress-choice\fP (\fB\-\-zc\fP) option unless you're sure of the choice in effect. For example: .RS 4 .IP @@ -2390,8 +2424,10 @@ Override the list of file suffixes that possible. Rsync sets the compression level on a per-file basis based on the file's suffix. If the compression algorithm has an "off" level (such as zlib/zlibx) then no compression occurs for those files. Other -algorithms have the level minimized to reduces the CPU usage as much as -possible. +algorithms that support changing the streaming level on-the-fly will have +the level minimized to reduces the CPU usage as much as possible for a +matching file. At this time, only zlib & zlibx compression support this +changing of levels on a per-file basis. .IP The \fBLIST\fP should be one or more file suffixes (without the dot) separated by slashes (\fB/\fP). You may specify an empty string to indicate that no files @@ -3128,6 +3164,11 @@ output buffer occurs. This may be fixed Record a file that can later be applied to another identical destination with \fB\-\-read-batch\fP. See the "BATCH MODE" section for details, and also the \fB\-\-only-write-batch\fP option. +.IP +This option overrides the negotiated checksum & compress lists and always +negotiates a choice based on old-school md5/md4/zlib choices. If you want +a more modern choice, use the \fB\-\-checksum-choice\fP (\fB\-\-cc\fP) and/or +\fB\-\-compress-choice\fP (\fB\-\-zc\fP) options. .IP "\fB\-\-only-write-batch=FILE\fP" Works like \fB\-\-write-batch\fP, except that no updates are made on the destination system when creating the batch. This lets you transport the @@ -3199,8 +3240,8 @@ it uses). These options also exist in the \fB\-\-daemon\fP mode section. .IP If rsync was complied without support for IPv6, the \fB\-\-ipv6\fP option will -have no effect. The \fBrsync\ \-V\fP output will contain "\fBno\ IPv6\fP" if is the -case. +have no effect. The \fBrsync\ \-\-version\fP output will contain "\fBno\ IPv6\fP" if +is the case. .IP "\fB\-\-checksum-seed=NUM\fP" Set the checksum seed to the integer NUM. This 4 byte checksum seed is included in each block and MD4 file checksum calculation (the more modern @@ -3292,8 +3333,8 @@ daemon). These options also exist in the regular rsync options section. .IP If rsync was complied without support for IPv6, the \fB\-\-ipv6\fP option will -have no effect. The \fBrsync\ \-V\fP output will contain "\fBno\ IPv6\fP" if is the -case. +have no effect. The \fBrsync\ \-\-version\fP output will contain "\fBno\ IPv6\fP" if +is the case. .IP "\fB\-\-help\fP, \fB\-h\fP" When specified after \fB\-\-daemon\fP, print a short help page describing the options available for starting an rsync daemon. @@ -4098,7 +4139,7 @@ Please report bugs! See the web site at .P .SH "VERSION" .P -This man page is current for version 3.2.1 of rsync. +This man page is current for version 3.2.2 of rsync. .P .SH "INTERNAL OPTIONS" .P diff -upN a/rsync.1.html b/rsync.1.html --- a/rsync.1.html +++ b/rsync.1.html @@ -394,6 +394,7 @@ detailed description below for a complet --max-delete=NUM don't delete more than NUM files --max-size=SIZE don't transfer any file larger than SIZE --min-size=SIZE don't transfer any file smaller than SIZE +--max-alloc=SIZE change a limit relating to memory alloc --partial keep partially transferred files --partial-dir=DIR put a partially transferred file into DIR --delay-updates put all updated files into place at end @@ -500,8 +501,6 @@ options since it normally means --

The output includes the default list of checksum algorithms, the default list of compression algorithms, a list of compiled-in capabilities, a link to the rsync web site, and some license/copyright info.

-

Repeat the option (-VV) to include some optimization info at the end of -the capabilities list.

--verbose, -v
@@ -583,7 +582,7 @@ errors that happen on a remote receiver file's data is fully sent. It is also helpful for debugging, since it helps to avoid overpopulating the protocol data with extra message data.

The option does not affect the remote side of a transfer without using ---remote-option -⁠-⁠ e.g. -M--msgs2stderr or {-M,}--msgs2stderr.

+--remote-option, e.g. -M--msgs2stderr or {-M,}--msgs2stderr.

Also keep in mind that connecting to a normal (non-remote-shell) daemon does not have a stderr channel to send messages back to the client side, so a modern rsync only allows the option on a remote-shell-run daemon.

@@ -663,7 +662,7 @@ is generated as the file is transferred, after-the-transfer verification has nothing to do with this option's before-the-transfer "Does this file need to be updated?" check.

The checksum used is auto-negotiated between the client and the server, but -can be overridden using either the --checksum-choice option or an +can be overridden using either the --checksum-choice (--cc) option or an environment variable that is discussed in that option's section.

@@ -1403,34 +1402,37 @@ comma-separated names are supplied, the checksums, and the second name affects the pre-transfer checksums (-c).

The checksum options that you may be able to use are:

    -
  • auto (the default)
  • -
  • xxh64 (aka xxhash)
  • +
  • auto (the default automatic choice)
  • +
  • xxh128
  • +
  • xxh3
  • +
  • xxh64 (aka xxhash)
  • md5
  • md4
  • none
-

Run rsync -V to see the default checksum list compiled into your version.

+

Run rsync --version to see the default checksum list compiled into your +version (which may differ from the list above).

If "none" is specified for the first (or only) name, the --whole-file option is forced on and no checksum verification is performed on the transferred data. If "none" is specified for the second (or only) name, the --checksum option cannot be used.

The "auto" option is the default, where rsync bases its algorithm choice on a negotiation between the client and the server as follows:

-

If both the client and the server are at least version 3.2.0, they will -exchange a list of checksum names and choose the first one in the list that -they have in common. This typically means that they will choose xxh64 if -they both support it and fall back to MD5. If one side of the transfer is -not new enough to support this checksum negotiation, then a value is chosen -based on the protocol version (which chooses between MD5 and various -flavors of MD4 based on protocol age).

-

You can also override the checksum using the RSYNC_CHECKSUM_LIST -environment variable by setting it to a space-separated list of checksum -names that you consider acceptable. If no common checksum is found, the -client exits with an error. This method does not allow you to specify the -transfer checksum separately from the pre-transfer checksum, and it ignores -"auto" and all unknown checksum names. If the remote rsync is not new -enough to handle a checksum negotiation list, the list is silently ignored -unless it contains the string "FAIL".

+

When both sides of the transfer are at least 3.2.0, rsync chooses the first +algorithm in the client's list of choices that is also in the server's list +of choices. If no common checksum choice is found, rsync exits with +an error. If the remote rsync is too old to support checksum negotiation, +a value is chosen based on the protocol version (which chooses between MD5 +and various flavors of MD4 based on protocol age).

+

The default order can be customized by setting the environment variable +RSYNC_CHECKSUM_LIST to a space-separated list of acceptable checksum names. +If the string contains a "&" character, it is separated into the "client +string & server string", otherwise the same string +applies to both. If the string (or string portion) contains no +non-whitespace characters, the default checksum list is used. This method +does not allow you to specify the transfer checksum separately from the +pre-transfer checksum, and it discards "auto" and all unknown checksum +names. A list with only invalid names results in a failed negotiation.

The use of the --checksum-choice option overrides this environment list.

@@ -1639,12 +1641,15 @@ size multiplier, and may be a fractional

This option is a transfer rule, not an exclude, so it doesn't affect the data that goes into the file-lists, and thus it doesn't affect deletions. It just limits the files that the receiver requests to be transferred.

-

The suffixes are as follows: "K" (or "KiB") is a kibibyte (1024), "M" (or -"MiB") is a mebibyte (1024*1024), and "G" (or "GiB") is a gibibyte -(1024*1024*1024). If you want the multiplier to be 1000 instead of 1024, -use "KB", "MB", or "GB". (Note: lower-case is also accepted for all -values.) Finally, if the suffix ends in either "+1" or "-⁠1", the value will -be offset by one byte in the indicated direction.

+

The accepted suffix letters are: B, K, G, T, and P for bytes, +kilobytes/kibibytes, megabytes/mebibytes, gigabytes/gibibytes, +terabytes/tebibytes, and petabytes/pebibytes. If you use a single-char +suffix or add-on "ib" to it (e.g. "G" or "GiB") then you get units that are +multiples of 1024. If you use a two-letter suffix that ends with a "B" +(e.g. "kb") then you get units that are multiples of 1000. The suffix +letters can be any mix of upper and lower-case that you want to use.

+

Finally, if the string ends with either "+1" or "-⁠1", it is offset by one +byte in the indicated direction. The largest possible value is 8192P-1.

Examples: --max-size=1.5mb-1 is 1499999 bytes, and --max-size=2g+1 is 2147483649 bytes.

Note that rsync versions prior to 3.1.0 did not allow --max-size=0.

@@ -1657,6 +1662,25 @@ the --max-size option for a

Note that rsync versions prior to 3.1.0 did not allow --min-size=0.

+
--max-alloc=SIZE
+

By default rsync limits an individual malloc/realloc to about 1GB in size. +For most people this limit works just fine and prevents a protocol error +causing rsync to request massive amounts of memory. However, if you have +many millions of files in a transfer, a large amount of server memory, and +you don't want to split up your transfer into multiple parts, you can +increase the per-allocation limit to something larger and rsync will +consume more memory.

+

Keep in mind that this is not a limit on the total size of allocated +memory. It is a sanity-check value for each individual allocation.

+

See the --max-size option for a description of how SIZE can be specified. +The default suffix if none is given is bytes.

+

You can set a default value using the environment variable RSYNC_MAX_ALLOC +using the same SIZE values as supported by this option. If the remote +rsync doesn't understand the --max-alloc option, you can override an +environmental value by specifying --max-alloc=1g, which will make rsync +avoid sending the option to the remote side (because "1G" is the default).

+
+
--block-size=SIZE, -B

This forces the block size used in rsync's delta-transfer algorithm to a fixed value. It is normally selected based on the size of each file being @@ -1962,8 +1986,9 @@ make sure it's disabled if you ever need that is older than that.

Rsync can also be configured (at build time) to have this option enabled by default (with is overridden by both the environment and the command-line). -Run rsync -V to check if this is the case, as it will display "default -protect-args" or "optional protect-args" depending on how it was compiled.

+Run rsync --version to check if this is the case, as it will display +"default protect-args" or "optional protect-args" depending on how it was +compiled.

This option will eventually become a new default setting at some as-yet-undetermined point in the future.

@@ -2133,27 +2158,37 @@ the -o option when sending

With this option, rsync compresses the file data as it is sent to the destination machine, which reduces the amount of data being transmitted -⁠-⁠ something that is useful over a slow connection.

-

The "zlib" compression method typically achieves better compression ratios -than can be achieved by using a compressing remote shell or a compressing -transport because it takes advantage of the implicit information in the -matching data blocks that are not explicitly sent over the connection. -This matching-data compression comes at a cost of CPU, though, and can be -disabled by using the "zlibx" compresson method instead. This can be -selected by repeating the -z option or specifying ---compress-choice=zlibx, but it only works if both sides of the transfer -are at least version 3.1.1.

-

Note that if you see an error about an option named --old-compress or ---new-compress, this is rsync trying to send the --compress-choice=zlib -or --compress-choice=zlibx option in a backward-compatible manner that -more rsync versions understand. This error indicates that the older rsync -version will not allow you to force the compression type.

-

See the --skip-compress option for the default list of file suffixes that -will not be compressed.

+

Rsync supports multiple compression methods and will choose one for you +unless you force the choice using the --compress-choice (--zc) option.

+

Run rsync --version to see the default compress list compiled into your +version.

+

When both sides of the transfer are at least 3.2.0, rsync chooses the first +algorithm in the client's list of choices that is also in the server's list +of choices. If no common compress choice is found, rsync exits with +an error. If the remote rsync is too old to support checksum negotiation, +its list is assumed to be "zlib".

+

The default order can be customized by setting the environment variable +RSYNC_COMPRESS_LIST to a space-separated list of acceptable compression +names. If the string contains a "&" character, it is separated into the +"client string & server string", otherwise the same string applies to both. +If the string (or string portion) contains no +non-whitespace characters, the default compress list is used. Any unknown +compression names are discarded from the list, but a list with only invalid +names results in a failed negotiation.

+

There are some older rsync versions that were configured to reject a -z +option and require the use of -zz because their compression library was +not compatible with the default zlib compression method. You can usually +ignore this weirdness unless the rsync server complains and tells you to +specify -zz.

+

See also the --skip-compress option for the default list of file suffixes +that will trasnferred with no (or minimal) compression.

--compress-choice=STR, --zc=STR
-

This option can be used to override the automatic selection of the -compression algorithm that is the default when --compress is used.

+

This option can be used to override the automatic negotiation of the +compression algorithm that occurs when --compress is used. The option +implies --compress unless "none" was specified, which instead implies +--no-compress.

The compression options that you may be able to use are:

  • zstd
  • @@ -2162,21 +2197,16 @@ compression algorithm that is the defaul
  • zlib
  • none
-

Run rsync -V to see the compress list compiled into your version.

-

The "zlibx" algorithm is given preference over "zlib" if both sides of the -transfer are at least version 3.2.0, otherwise it will choose "zlib" unless -you override it via something like -zz. These 2 algorithms are the stame -except that "zlibx" does not try to include matched data that was not -transferred in the compression computations.

-

If "none" is specified, that is equivalent to using --no-compress.

-

This option implies --compress unless "none" was specified.

-

You can also override the compression negotiation using the -RSYNC_COMPRESS_LIST environment variable by setting it to a space-separated -list of compression names that you consider acceptable. If no common -compress choice is found, the client exits with an error. It ignores -"auto" and all unknown compression names. If the remote rsync is not new -enough to handle a compression negotiation list, the list is silently -ignored unless it contains the string "FAIL".

+

Run rsync --version to see the default compress list compiled into your +version (which may differ from the list above).

+

Note that if you see an error about an option named --old-compress or +--new-compress, this is rsync trying to send the --compress-choice=zlib +or --compress-choice=zlibx option in a backward-compatible manner that +more rsync versions understand. This error indicates that the older rsync +version on the server will not allow you to force the compression type.

+

Note that the "zlibx" compression algorithm is just the "zlib" algorithm +with matched data excluded from the compression stream (to try to make it +more compatible with an external zlib implementation).

--compress-level=NUM, --zl=NUM
@@ -2186,8 +2216,8 @@ as the level chosen is not a "don't algorithm that is in effect (e.g. zlib compression treats level 0 as "off").

The level values vary depending on the checksum in effect. Because rsync -will negotiate a checksum choice by default when the remote rsync is new -enough, it can be good to combine this option with a --compress-choice +will negotiate a checksum choice by default (when the remote rsync is new +enough), it can be good to combine this option with a --compress-choice (--zc) option unless you're sure of the choice in effect. For example:

rsync -aiv --zc=zstd --zl=22 host:src/ dest/
@@ -2214,8 +2244,10 @@ choice in effect).

possible. Rsync sets the compression level on a per-file basis based on the file's suffix. If the compression algorithm has an "off" level (such as zlib/zlibx) then no compression occurs for those files. Other -algorithms have the level minimized to reduces the CPU usage as much as -possible.

+algorithms that support changing the streaming level on-the-fly will have +the level minimized to reduces the CPU usage as much as possible for a +matching file. At this time, only zlib & zlibx compression support this +changing of levels on a per-file basis.

The LIST should be one or more file suffixes (without the dot) separated by slashes (/). You may specify an empty string to indicate that no files should be skipped.

@@ -2895,6 +2927,10 @@ output buffer occurs. This may be fixed

Record a file that can later be applied to another identical destination with --read-batch. See the "BATCH MODE" section for details, and also the --only-write-batch option.

+

This option overrides the negotiated checksum & compress lists and always +negotiates a choice based on old-school md5/md4/zlib choices. If you want +a more modern choice, use the --checksum-choice (--cc) and/or +--compress-choice (--zc) options.

--only-write-batch=FILE
@@ -2968,8 +3004,8 @@ the "--rsh SHELL -4&qu it uses).

These options also exist in the --daemon mode section.

If rsync was complied without support for IPv6, the --ipv6 option will -have no effect. The rsync -V output will contain "no IPv6" if is the -case.

+have no effect. The rsync --version output will contain "no IPv6" if +is the case.

--checksum-seed=NUM
@@ -3081,8 +3117,8 @@ using the port, try specifying --i daemon).

These options also exist in the regular rsync options section.

If rsync was complied without support for IPv6, the --ipv6 option will -have no effect. The rsync -V output will contain "no IPv6" if is the -case.

+have no effect. The rsync --version output will contain "no IPv6" if +is the case.

--help, -h
@@ -3726,7 +3762,7 @@ values

see also the comments on the --delete option

Please report bugs! See the web site at https://rsync.samba.org/.

VERSION

-

This man page is current for version 3.2.1 of rsync.

+

This man page is current for version 3.2.2 of rsync.

INTERNAL OPTIONS

The options --server and --sender are used internally by rsync, and should never be typed by a user under normal circumstances. Some awareness of these @@ -3755,5 +3791,5 @@ people have later contributed to it. It Davison.

Mailing lists for support and development are available at https://lists.samba.org/.

-

22 Jun 2020

+

04 Jul 2020

diff -upN a/rsyncd.conf.5 b/rsyncd.conf.5 --- a/rsyncd.conf.5 +++ b/rsyncd.conf.5 @@ -1,4 +1,4 @@ -.TH "rsyncd.conf" "5" "22 Jun 2020" "rsyncd.conf 3.2.1" "User Commands" +.TH "rsyncd.conf" "5" "04 Jul 2020" "rsyncd.conf 3.2.2" "User Commands" .P .SH "NAME" .P @@ -397,6 +397,20 @@ disabled. .IP Helpful hint: you probably want to specify "refuse options = delete" for a write-only module. +.IP "\fBopen\ noatime\fP" +When set to True, this parameter tells the rsync daemon to open files with +the O_NOATIME flag +(on systems that support it) to avoid changing the access time of the files +that are being transferred. If your OS does not support the O_NOATIME flag +then rsync will silently ignore this option. Note also that some +filesystems are mounted to avoid updating the atime on read access even +without the O_NOATIME flag being set. +.IP +When set to False, this parameters ensures that files on the server are not +opened with O_NOATIME. +.IP +When set to Unset (the default) the user controls the setting via +\fB\-\-open-noatime\fP. .IP "\fBlist\fP" This parameter determines whether this module is listed when the client asks for a listing of available modules. In addition, if this is false, @@ -900,19 +914,19 @@ Here are all the options that are not ma .IP o \fB\-\-rsh\fP, \fB\-e\fP: Required to convey compatibility flags to the server. .IP o -\fB\-\-log-format\fP: This is required to convey things like -\fB\-\-itemize-changes\fP to a remote receiver. Is an older name for -\fB\-\-out-format\fP that is still passed to the server for improved backward -compatibility and should not be confused with \fB\-\-log-file-format\fP. +\fB\-\-out-format\fP: This is required to convey output behavior to a remote +receiver. While rsync passes the older alias \fB\-\-log-format\fP for +compatibility reasons, this options should not be confused with +\fB\-\-log-file-format\fP. .IP o \fB\-\-sender\fP: Use "write only" parameter instead of refusing this. .IP o \fB\-\-dry-run\fP, \fB\-n\fP: Who would want to disable this? .IP o -\fB\-\-protect-args\fP, \fB\-n\fP: This actually makes transfers safer. +\fB\-\-protect-args\fP, \fB\-s\fP: This actually makes transfers safer. .IP o -\fB\-\-from0\fP, \fB\-0\fP: Make it easier to accept/refuse \fB\-\-files-from\fP without -affecting this modifier. +\fB\-\-from0\fP, \fB\-0\fP: Makes it easier to accept/refuse \fB\-\-files-from\fP without +affecting this helpful modifier. .IP o \fB\-\-iconv\fP: This is auto-disabled based on "charset" parameter. .IP o @@ -1134,7 +1148,7 @@ stream { listen [::]:874 ssl; ssl_certificate /etc/letsencrypt/example.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem + ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem; proxy_pass localhost:873; proxy_protocol on; # Requires "proxy protocol = true" @@ -1217,7 +1231,7 @@ https://rsync.samba.org/. .P .SH "VERSION" .P -This man page is current for version 3.2.1 of rsync. +This man page is current for version 3.2.2 of rsync. .P .SH "CREDITS" .P diff -upN a/rsyncd.conf.5.html b/rsyncd.conf.5.html --- a/rsyncd.conf.5.html +++ b/rsyncd.conf.5.html @@ -410,6 +410,20 @@ disabled.

write-only module.

+
open noatime
+

When set to True, this parameter tells the rsync daemon to open files with +the O_NOATIME flag +(on systems that support it) to avoid changing the access time of the files +that are being transferred. If your OS does not support the O_NOATIME flag +then rsync will silently ignore this option. Note also that some +filesystems are mounted to avoid updating the atime on read access even +without the O_NOATIME flag being set.

+

When set to False, this parameters ensures that files on the server are not +opened with O_NOATIME.

+

When set to Unset (the default) the user controls the setting via +--open-noatime.

+
+
list

This parameter determines whether this module is listed when the client asks for a listing of available modules. In addition, if this is false, @@ -873,15 +887,15 @@ explicitly accepted with "!wr

  • --server: Required for rsync to even work.
  • --rsh, -e: Required to convey compatibility flags to the server.
  • -
  • --log-format: This is required to convey things like ---itemize-changes to a remote receiver. Is an older name for ---out-format that is still passed to the server for improved backward -compatibility and should not be confused with --log-file-format.
  • +
  • --out-format: This is required to convey output behavior to a remote +receiver. While rsync passes the older alias --log-format for +compatibility reasons, this options should not be confused with +--log-file-format.
  • --sender: Use "write only" parameter instead of refusing this.
  • --dry-run, -n: Who would want to disable this?
  • -
  • --protect-args, -n: This actually makes transfers safer.
  • -
  • --from0, -0: Make it easier to accept/refuse --files-from without -affecting this modifier.
  • +
  • --protect-args, -s: This actually makes transfers safer.
  • +
  • --from0, -0: Makes it easier to accept/refuse --files-from without +affecting this helpful modifier.
  • --iconv: This is auto-disabled based on "charset" parameter.
  • --no-iconv: Most transfers use this option.
  • --checksum-seed: Is a fairly rare, safe option.
  • @@ -1057,7 +1071,7 @@ backend be_rsync listen [::]:874 ssl; ssl_certificate /etc/letsencrypt/example.com/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem + ssl_certificate_key /etc/letsencrypt/example.com/privkey.pem; proxy_pass localhost:873; proxy_protocol on; # Requires "proxy protocol = true" @@ -1122,7 +1136,7 @@ susan:herpass

    Please report bugs! The rsync bug tracking system is online at https://rsync.samba.org/.

    VERSION

    -

    This man page is current for version 3.2.1 of rsync.

    +

    This man page is current for version 3.2.2 of rsync.

    CREDITS

    rsync is distributed under the GNU General Public License. See the file COPYING for details.

    @@ -1139,5 +1153,5 @@ Thanks to Karsten Thygesen for his many later contributed to it.

    Mailing lists for support and development are available at https://lists.samba.org/.

    -

    22 Jun 2020

    +

    04 Jul 2020