[Date Prev][Date Next] [Thread Prev][Thread Next] [Date Index] [Thread Index]

Re: dpkg speed up



On Sat, 11 Nov 2000, Adam Heath wrote:

> (Sorry, don't have a patch for this yet, I'll generate a combined one later
> today, after I wake up).
> 
> In dpkg-deb/extract.c, there is a copy loop, that reads data from the .deb ar,
> and writes it to a pipe(this is done inside a fork).  This copy loop was doing
> it one byte at a time!
> 
> I also have dpkg-deb using zlib(I compile statically, so no external
> dependencies).  This also gave a speedup.
> 
> Extraction of 13m tetex-base, after the deb was in the disk cache.

Here are the first few patches.  The first one adds another option to
do_fd_copy(a limit parameter).  The second depends on the first, and does the
main speedup in extract.  I still need to polish up the zlib
integration(which, I'll state again, because it is being compiled static, is
only used for extraction, to limit size.  The goal is to eventually maybe have
a trimmed down dpkg).


----BEGIN GEEK CODE BLOCK----
Version: 3.12
GCS d- s: a-- c+++ UL++++ P+ L++++ !E W+ M o+ K- W--- !O M- !V PS--
PE++ Y+ PGP++ t* 5++ X+ tv b+ D++ G e h*! !r z?
-----END GEEK CODE BLOCK-----
----BEGIN PGP INFO----
Adam Heath <doogie@debian.org>        Finger Print | KeyID
67 01 42 93 CA 37 FB 1E    63 C9 80 1D 08 CF 84 0A | DE656B05 PGP
AD46 C888 F587 F8A3 A6DA  3261 8A2C 7DC2 8BD4 A489 | 8BD4A489 GPG
-----END PGP INFO-----
Binary files dpkg-1.7.1.2/core and dpkg-1.7.1.3/core differ
diff -ruN dpkg-1.7.1.2/dpkg-deb/build.c dpkg-1.7.1.3/dpkg-deb/build.c
--- dpkg-1.7.1.2/dpkg-deb/build.c	Fri Nov 10 02:09:24 2000
+++ dpkg-1.7.1.3/dpkg-deb/build.c	Sat Nov 11 06:20:13 2000
@@ -364,7 +364,7 @@
   }                
                 
   if (lseek(gzfd,0,SEEK_SET)) ohshite(_("failed to rewind tmpfile (control)"));
-  do_fd_copy(gzfd, fileno(ar), _("control"));
+  do_fd_copy(gzfd, fileno(ar), -1, _("control"));
 
   /* Control is done, now we need to archive the data. Start by creating
    * a new temporary file. Immediately unlink the temporary file so others
@@ -403,7 +403,7 @@
     combuf = strdup("-9c");
     if(compression != NULL) {
       if(*compression == '0') {
-	do_fd_copy(0, 1, _("no compression copy loop"));
+	do_fd_copy(0, 1, -1, _("no compression copy loop"));
 	exit(0);
       }
       combuf[1] = *compression;
@@ -457,7 +457,7 @@
       werr(debar);
 
     if (lseek(gzfd,0,SEEK_SET)) ohshite(_("failed to rewind tmpfile (data)"));
-    do_fd_copy(gzfd, fileno(ar), _("cat (data)"));
+    do_fd_copy(gzfd, fileno(ar), -1, _("cat (data)"));
 
     if (datastab.st_size & 1)
       if (putc('\n',ar) == EOF)
diff -ruN dpkg-1.7.1.2/dpkg-deb/info.c dpkg-1.7.1.3/dpkg-deb/info.c
--- dpkg-1.7.1.2/dpkg-deb/info.c	Fri Dec 24 10:28:11 1999
+++ dpkg-1.7.1.3/dpkg-deb/info.c	Sat Nov 11 06:20:53 2000
@@ -90,7 +90,7 @@
   while ((component= *argv++) != 0) {
     co= fopen(component,"r");
     if (co) {
-      do_fd_copy(fileno(co), 1, _("info_spew"));
+      do_fd_copy(fileno(co), 1, -1, _("info_spew"));
     } else if (errno == ENOENT) {
       if (fprintf(stderr, _("dpkg-deb: `%.255s' contains no control component `%.255s'\n"),
                   debar, component) == EOF) werr("stderr");
diff -ruN dpkg-1.7.1.2/include/dpkg.h.in dpkg-1.7.1.3/include/dpkg.h.in
--- dpkg-1.7.1.2/include/dpkg.h.in	Sun Oct  1 13:45:47 2000
+++ dpkg-1.7.1.3/include/dpkg.h.in	Sat Nov 11 06:14:18 2000
@@ -199,7 +199,7 @@
 void checksubprocerr(int status, const char *description, int sigpipeok);
 void waitsubproc(pid_t pid, const char *description, int sigpipeok);
 
-int do_fd_copy(int fd1, int fd2, char *desc);
+int do_fd_copy(int fd1, int fd2, int limit, char *desc);
 
 extern volatile int onerr_abort;
 
diff -ruN dpkg-1.7.1.2/lib/mlib.c dpkg-1.7.1.3/lib/mlib.c
--- dpkg-1.7.1.2/lib/mlib.c	Fri Dec 24 10:28:11 1999
+++ dpkg-1.7.1.3/lib/mlib.c	Sat Nov 11 06:14:57 2000
@@ -123,9 +123,9 @@
   checksubprocerr(status,description,sigpipeok);
 }
 
-int do_fd_copy(int fd1, int fd2, char *desc) {
+int do_fd_copy(int fd1, int fd2, int limit, char *desc) {
     char *buf, *sbuf;
-    int count;
+    int count, bufsize = 32768;
     char *er_msg_1 = _("failed to allocate buffer for copy (%s)");
     char *er_msg_2 = _("failed in copy on write (%s)");
     char *er_msg_3 = _("failed in copy on read (%s)");
@@ -137,7 +137,9 @@
     snprintf(sbuf, count, er_msg_1, desc);
     sbuf[count-1] = 0;
 
-    buf = malloc(32768);
+    if((limit != -1) && (limit < bufsize))
+	bufsize = limit;
+    buf = malloc(bufsize);
     if(buf == NULL)
 	ohshite(sbuf);
     free(sbuf);
@@ -149,10 +151,15 @@
     snprintf(sbuf, count, er_msg_2, desc);
     sbuf[count-1] = 0;
 
-    while((count = read(fd1, buf, 32768)) > 0)
+    while((count = read(fd1, buf, bufsize)) > 0) {
 	if(write(fd2, buf, count) < count)
-		ohshite(sbuf);
-
+	    ohshite(sbuf);
+	if(limit != -1) {
+	    limit -= count;
+	    if(limit < bufsize)
+		bufsize = limit;
+	}
+    }
     free(sbuf);
     count = strlen(er_msg_3) + strlen(desc) + 1;
     sbuf = malloc(count);
diff -ruN dpkg-1.7.1.2/lib/showcright.c dpkg-1.7.1.3/lib/showcright.c
--- dpkg-1.7.1.2/lib/showcright.c	Tue Jun 27 17:11:30 2000
+++ dpkg-1.7.1.3/lib/showcright.c	Sat Nov 11 06:21:20 2000
@@ -29,6 +29,6 @@
   int fd;
   fd= open(COPYINGFILE,O_RDONLY);
   if (fd < 0) ohshite(_("cannot open GPL file "));
-  do_fd_copy(fd, 1, "showcopyright");
+  do_fd_copy(fd, 1, -1, "showcopyright");
   exit(0);
 }
diff -ruN dpkg-1.7.1.2/dpkg-deb/extract.c dpkg-1.7.1.3/dpkg-deb/extract.c
--- dpkg-1.7.1.2/dpkg-deb/extract.c	Sat Oct 28 18:35:11 2000
+++ dpkg-1.7.1.3/dpkg-deb/extract.c	Sat Nov 11 06:31:20 2000
@@ -245,13 +245,8 @@
     m_pipe(p1);
     if (!(c1= m_fork())) {
       close(p1[0]);
-      if (!(pi= fdopen(p1[1],"w"))) ohshite(_("failed to fdopen p1 in copy"));
-      while (memberlen > 0) {
-        if ((c= getc(ar)) == EOF) readfail(ar,debar,_("member data"));
-        if (putc(c,pi) == EOF) ohshite(_("failed to write to pipe in copy"));
-        memberlen--;
-      }
-      if (fclose(pi) == EOF) ohshite(_("failed to close pipe in copy"));
+      do_fd_copy(fileno(ar), p1[1], memberlen, _("failed to write to pipe in copy"));
+      if (close(p1[1]) == EOF) ohshite(_("failed to close pipe in copy"));
       exit(0);
     }
     close(p1[1]);

Reply to: