Bug#212734: cookie support in http method (Progeny)
Package: apt
Version: 0.5.9
Severity: wishlist
Tags: patch
The following patch adds cookie support to apt's http method.
Cookie support is turned on with Acquire::http::Use-Cookies (default
"true" in the patch; feel free to change this). Cookies are stored in
the file mentioned in Acquire::http::Cookie-File, which by default is
"/var/lib/apt/cookies.txt". It should be possible to set Cookie-File to
"/dev/null", which gives you the equivalent of transient cookies. The
cookies.txt file format is the same as that of most browsers.
I know this patch may be a bit controversial, but it really helps
Progeny with some of the things we're doing. We don't mind maintaining
this as a separate effort, but we want to keep our difference from
mainline apt as small as possible. So, if you have objections to any of
the decisions I've made in writing this patch, please let me know so we
can work out solutions for them.
Index: methods/http.cc
===================================================================
--- methods/http.cc (.../tags/apt-debian-0.5.9) (revision 37)
+++ methods/http.cc (.../branches/apt-resolve-0.5.9-cookie) (revision
37)
@@ -37,9 +37,15 @@
#include <unistd.h>
#include <signal.h>
#include <stdio.h>
+#include <stdlib.h>
+#include <ctype.h>
#include <errno.h>
#include <string.h>
#include <iostream>
+#include <fstream>
+#include <sstream>
+#include <map>
+#include <algorithm>
// Internet stuff
#include <netdb.h>
@@ -51,6 +57,14 @@
/*}}}*/
using namespace std;
+// Functor for tolower() that allows it to be used by the transform
+// algorithm.
+
+struct ToLower
+{
+ char operator() (char c) const { return std::tolower(c); }
+};
+
string HttpMethod::FailFile;
int HttpMethod::FailFd = -1;
time_t HttpMethod::FailTime = 0;
@@ -58,6 +72,449 @@
unsigned long TimeOut = 120;
bool Debug = false;
+// CookieJar::CookieJar - Cookie manager /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+CookieJar::CookieJar()
+{
+ IsModified = false;
+ IsEnabled = false;
+ LockFd = -1;
+}
+ /*}}}*/
+// CookieJar::Compare - Compare a cookie to see if it matches a
URI. /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+bool CookieJar::Compare(const URI &Item, const Cookie &CookieRec)
+{
+ // All expired cookies are bad
+ if (CookieRec.Expires < time(NULL))
+ return false;
+
+ // If the cookie was marked secure, don't let it out except over
https
+#ifdef WITH_SSL
+ if (CookieRec.Secure && Item.Access != "https")
+ return false;
+#else
+ if (CookieRec.Secure)
+ return false;
+#endif
+
+ // Check that the hostname matches the cookie's domain
+ if (CookieRec.Access == true)
+ {
+ string::size_type DomainLength = CookieRec.Domain.length();
+ if (DomainLength >= Item.Host.length())
+ return false;
+
+ string DomainPart = Item.Host.substr(Item.Host.length() -
DomainLength,
+ DomainLength);
+ if (DomainPart != CookieRec.Domain)
+ return false;
+ }
+ else
+ {
+ if (CookieRec.Domain != Item.Host)
+ return false;
+ }
+
+ // Check that the URI path is in the allowed cookie path
+ if (CookieRec.Path.length() > Item.Path.length())
+ return false;
+ if (CookieRec.Path != Item.Path.substr(0, CookieRec.Path.length()))
+ return false;
+
+ // All tests check out - must be OK
+ return true;
+}
+ /*}}}*/
+// CookieJar::AddCookie - Add a new cookie to the list. /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+void CookieJar::AddCookie(const Cookie &CookieRec)
+{
+ vector<Cookie>::iterator CookieItem;
+
+ // Replace cookies in the new cookie list.
+ for (CookieItem = NewCookieList.begin(); CookieItem !=
NewCookieList.end();
+ CookieItem++)
+ {
+ if (CookieItem->Domain == CookieRec.Domain &&
+ CookieItem->Name == CookieRec.Name)
+ {
+ *CookieItem = CookieRec;
+ return;
+ }
+ }
+
+ // Brand new - add to the new cookie list.
+ NewCookieList.push_back(CookieRec);
+}
+ /*}}}*/
+// CookieJar::Lock - Lock the cookie file. /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+bool CookieJar::Lock()
+{
+ bool IsLocked = false;
+ string LockFileName = CookieFileName + ".lock";
+
+ for (int Counter = 0; Counter < 3 && !IsLocked; Counter++)
+ {
+ if (Counter != 0)
+ sleep(1);
+ LockFd = GetLock(LockFileName);
+ IsLocked = (LockFd != -1);
+ }
+
+ return IsLocked;
+}
+ /*}}}*/
+// CookieJar::UnLock - Unock the cookie file. /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+void CookieJar::UnLock()
+{
+ string LockFileName = CookieFileName + ".lock";
+
+ if (LockFd != -1)
+ {
+ close(LockFd);
+ unlink(LockFileName.c_str());
+ LockFd = -1;
+ }
+}
+ /*}}}*/
+// CookieJar::Configure - Configure the cookie jar /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+void CookieJar::Configure()
+{
+ IsEnabled = _config->FindB("Acquire::http::Use-Cookies",true);
+ if (IsEnabled)
+ CookieFileName = _config->Find("Acquire::http::Cookie-File",
+ "/var/lib/apt/cookies.txt");
+}
+ /*}}}*/
+// CookieJar::ReadCookies - Read the cookie file /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+bool CookieJar::ReadCookies(bool DoLock)
+{
+ Cookie NewCookie;
+
+ // Silently do nothing if cookies are disabled
+ if (!IsEnabled)
+ return true;
+
+ // If the cookie file doesn't exist, don't load any
+ if (FileExists(CookieFileName) == false)
+ return true;
+
+ // Lock the cookie file to prevent updates
+ if (DoLock == true)
+ if (Lock() == false)
+ return false;
+
+ // Read the cookie file
+ CookieList.clear();
+ ifstream CookieFile(CookieFileName.c_str());
+ string line;
+ while (getline(CookieFile, line))
+ {
+ // Skip comments and blank lines
+ if (line[0] == '#')
+ continue;
+ if (line.find_first_not_of(" \t\n") == string::npos)
+ continue;
+
+ // Split the line into its constituent tokens
+ istringstream Splitter(line);
+ string AccessStr, SecureStr;
+
+ Splitter >> NewCookie.Domain >> AccessStr >> NewCookie.Path
+ >> SecureStr >> NewCookie.Expires >> NewCookie.Name
+ >> NewCookie.Value;
+
+ // If we couldn't tokenize the line, something's wrong
+ if (Splitter.fail())
+ return false;
+
+ // If the cookie has expired, then skip it
+ if (NewCookie.Expires < time(NULL))
+ {
+ IsModified = true;
+ continue;
+ }
+
+ // Convert the boolean string values into true booleans
+ if (AccessStr == "TRUE")
+ NewCookie.Access = true;
+ else
+ NewCookie.Access = false;
+ if (SecureStr == "TRUE")
+ NewCookie.Secure = true;
+ else
+ NewCookie.Secure = false;
+
+ // Integrate the new cookie into the list
+ CookieList.push_back(NewCookie);
+ }
+
+ // Clean up.
+ CookieFile.close();
+ if (DoLock == true)
+ UnLock();
+}
+ /*}}}*/
+// CookieJar::DoRealWrite - Write cookies to the cookie file /*{{{*/
+//
---------------------------------------------------------------------
+/* This function does the real work of writing the cookies to the
cookie
+ file. It's separated this way because of an optimizer bug in gcc
+ 2.95. */
+bool CookieJar::DoRealWrite()
+{
+ // Lock the cookie file
+ if (Lock() == false)
+ return false;
+
+ /* Re-read the cookie file before trying to write it, so we don't
+ lose any updates. */
+ ReadCookies(false);
+
+ // Get list of unmodified cookies to write
+ vector<Cookie> WriteCookieList;
+ vector<Cookie>::iterator CookieItem;
+ vector<Cookie>::iterator NewCookieItem;
+ for (CookieItem = CookieList.begin(); CookieItem !=
CookieList.end();
+ CookieItem++)
+ {
+ // Don't write expired cookies.
+ if (CookieItem->Expires < time(0))
+ continue;
+
+ // Don't write cookies that have changed
+ bool CookieModified = false;
+ for (NewCookieItem = NewCookieList.begin();
+ NewCookieItem != NewCookieList.end();
+ NewCookieItem++)
+ if (NewCookieItem->Name == CookieItem->Name &&
+ NewCookieItem->Domain == CookieItem->Domain)
+ {
+ CookieModified = true;
+ break;
+ }
+ if (CookieModified == true)
+ continue;
+
+ // We got one
+ WriteCookieList.push_back(*CookieItem);
+ }
+
+ // Tack on all modified cookies
+ WriteCookieList.insert(WriteCookieList.end(), NewCookieList.begin(),
+ NewCookieList.end());
+
+ // Open the cookie file, or fail
+ ofstream CookieFile(CookieFileName.c_str(), ios::out | ios::trunc);
+ if (CookieFile.is_open() == false)
+ {
+ UnLock();
+ return false;
+ }
+
+ // Write cookie list
+ CookieFile << "# Cookie file for apt\n\n";
+
+ for (CookieItem = WriteCookieList.begin();
+ CookieItem != WriteCookieList.end();
+ CookieItem++)
+ {
+ CookieFile << CookieItem->Domain << "\t";
+ if (CookieItem->Access)
+ CookieFile << "TRUE\t";
+ else
+ CookieFile << "FALSE\t";
+ CookieFile << CookieItem->Path << "\t";
+ if (CookieItem->Secure)
+ CookieFile << "TRUE\t";
+ else
+ CookieFile << "FALSE\t";
+ CookieFile << CookieItem->Expires << "\t" << CookieItem->Name <<
"\t"
+ << CookieItem->Value << endl;
+ }
+
+ // Release the cookie file
+ CookieFile.close();
+ UnLock();
+
+ /* Now the written list is the unmodified list, and there are no
+ unmodified cookies. */
+ CookieList.clear();
+ CookieList = WriteCookieList;
+ NewCookieList.clear();
+
+ // Clean up
+ IsModified = false;
+ return true;
+}
+ /*}}}*/
+// CookieJar::GetCookies - Get cookie field /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+bool CookieJar::GetCookies(const URI &Srv, string &Data)
+{
+ // Silently do nothing if cookies are disabled
+ if (!IsEnabled)
+ return false;
+
+ /* Write relevant cookie info into a string suitable as the value
+ for a HTTP Cookie header line. */
+ vector<Cookie>::const_iterator CookieItem;
+ vector<Cookie>::const_iterator NewCookieItem;
+ string CookieStr;
+
+ for (CookieItem = NewCookieList.begin(); CookieItem !=
NewCookieList.end();
+ CookieItem++)
+ if (Compare(Srv, *CookieItem))
+ CookieStr += CookieItem->Name + "=" + CookieItem->Value + "; ";
+ for (CookieItem = CookieList.begin(); CookieItem !=
CookieList.end();
+ CookieItem++)
+ {
+ for (NewCookieItem = NewCookieList.begin();
+ NewCookieItem != NewCookieList.end();
+ NewCookieItem++)
+ if (CookieItem->Name == NewCookieItem->Name &&
+ CookieItem->Domain == NewCookieItem->Domain)
+ continue;
+
+ if (Compare(Srv, *CookieItem))
+ CookieStr += CookieItem->Name + "=" + CookieItem->Value + "; ";
+ }
+
+ // If we didn't find any cookies, don't bother
+ if (CookieStr.length() == 0)
+ return false;
+ else
+ {
+ Data = CookieStr.substr(0, CookieStr.length() - 2);
+ return true;
+ }
+}
+ /*}}}*/
+// CookieJar::SetCookie - Add or change a cookie /*{{{*/
+//
---------------------------------------------------------------------
+/* */
+bool CookieJar::SetCookie(const URI &Item, const string &Data)
+{
+ Cookie NewCookieRec;
+ bool CookieSaved = false;
+
+ // Tokenize the data string
+ istringstream DataStream(Data);
+ string Token;
+ while (getline(DataStream, Token, ';'))
+ {
+ string Name, Value, FieldName;
+ string::size_type Equal;
+
+ // Extract the name and value from the token
+ Equal = Token.find("=");
+ if (Equal == string::npos)
+ {
+ Name = Token;
+ Value = "";
+ }
+ else
+ {
+ Name = Token.substr(0, Equal);
+ Value = Token.substr(Equal + 1, Token.length() - Equal - 1);
+ }
+
+ // Strip whitespace from the beginning and end of the name and
value
+ while (Name.begin() != Name.end() && *(Name.begin()) == ' ')
+ Name.erase(Name.begin());
+ while (Name.begin() != Name.end() && *(Name.end() - 1) == ' ')
+ Name.erase(Name.end() - 1);
+ while (Value.begin() != Value.end() && *(Value.begin()) == ' ')
+ Value.erase(Value.begin());
+ while (Value.begin() != Value.end() && *(Value.end() - 1) == ' ')
+ Value.erase(Value.end() - 1);
+
+ // Lowercase all field names for comparison purposes
+ FieldName = Name;
+ transform(Name.begin(), Name.end(), FieldName.begin(),
ToLower());
+
+ // Expiration time
+ if (FieldName == "expires")
+ {
+ if (StrToTime(Value, NewCookieRec.Expires) == false)
+ return false;
+ }
+
+ // Max-Age; alternative mechanism to Expires
+ else if (FieldName == "max-age")
+ {
+ time_t interval = (time_t)strtoul(Value.c_str(), 0, 0);
+ NewCookieRec.Expires = time(0) + interval - 1;
+ }
+
+ // Path qualification
+ else if (FieldName == "path")
+ NewCookieRec.Path = Value;
+
+ // Domain limiter
+ else if (FieldName == "domain")
+ {
+ NewCookieRec.Domain = Value;
+ if (Value[0] == '.')
+ NewCookieRec.Access = false;
+ else
+ NewCookieRec.Access = true;
+ }
+
+ // Secure flag
+ else if (FieldName == "secure")
+ NewCookieRec.Secure = true;
+
+ // New cookie value
+ else
+ {
+ // Save the old cookie, if there is one
+ if (!NewCookieRec.Name.empty())
+ {
+ AddCookie(NewCookieRec);
+ CookieSaved = true;
+ }
+
+ // Create new cookie
+ NewCookieRec.Name = Name;
+ NewCookieRec.Value = Value;
+ NewCookieRec.Expires = time(0) + 86400;
+ NewCookieRec.Domain = Item.Host;
+ NewCookieRec.Path = "/";
+ NewCookieRec.Access = false;
+ if (Item.Access == "https")
+ NewCookieRec.Secure = true;
+ else
+ NewCookieRec.Secure = false;
+ }
+ }
+
+ // Save the last cookie, if there is one
+ if (!NewCookieRec.Name.empty())
+ {
+ AddCookie(NewCookieRec);
+ CookieSaved = true;
+ }
+
+ // Set IsModified if needed
+ if (CookieSaved)
+ IsModified = true;
+
+ return CookieSaved;
+}
+ /*}}}*/
// CircleBuf::CircleBuf - Circular input buffer /*{{{*/
//
---------------------------------------------------------------------
/* */
@@ -250,9 +707,9 @@
// ServerState::ServerState - Constructor /*{{{*/
//
---------------------------------------------------------------------
/* */
-ServerState::ServerState(URI Srv,HttpMethod *Owner) : Owner(Owner),
- In(64*1024), Out(4*1024),
- ServerName(Srv)
+ServerState::ServerState(URI Srv,HttpMethod *Owner,CookieJar &Cookies_)
:
+ Owner(Owner), In(64*1024), Out(4*1024),
+ ServerName(Srv), Cookies(Cookies_)
{
Reset();
}
@@ -591,6 +1048,12 @@
return true;
}
+ if (stringcasecmp(Tag,"Set-Cookie:") == 0)
+ {
+ Cookies.SetCookie(ServerName,Val);
+ return true;
+ }
+
return true;
}
/*}}}*/
@@ -672,7 +1135,11 @@
if (Uri.User.empty() == false || Uri.Password.empty() == false)
Req += string("Authorization: Basic ") +
- Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
+ Base64Encode(Uri.User + ":" + Uri.Password) + "\r\n";
+
+ string CookieStr;
+ if (Cookies.GetCookies(Uri, CookieStr) == true)
+ Req += string("Cookie: ") + CookieStr + "\r\n";
Req += "User-Agent: Debian APT-HTTP/1.3\r\n\r\n";
@@ -985,6 +1452,9 @@
PipelineDepth = _config->FindI("Acquire::http::Pipeline-Depth",
PipelineDepth);
Debug = _config->FindB("Debug::Acquire::http",false);
+
+ Cookies.Configure();
+ Cookies.ReadCookies();
return true;
}
@@ -1022,7 +1492,7 @@
if (Server == 0 || Server->Comp(Queue->Uri) == false)
{
delete Server;
- Server = new ServerState(Queue->Uri,this);
+ Server = new ServerState(Queue->Uri,this,Cookies);
}
/* If the server has explicitly said this is the last connection
Index: methods/http.h
===================================================================
--- methods/http.h (.../tags/apt-debian-0.5.9) (revision 37)
+++ methods/http.h (.../branches/apt-resolve-0.5.9-cookie) (revision 37)
@@ -14,12 +14,53 @@
#define MAXLEN 360
#include <iostream>
+#include <string>
+#include <vector>
using std::cout;
using std::endl;
class HttpMethod;
+class CookieJar
+{
+ struct Cookie
+ {
+ string Domain;
+ string Path;
+ string Name;
+ string Value;
+ time_t Expires;
+ bool Secure;
+ bool Access;
+ };
+
+ vector<Cookie> CookieList;
+ vector<Cookie> NewCookieList;
+
+ string CookieFileName;
+ bool IsEnabled;
+ bool IsModified;
+ int LockFd;
+
+ bool Compare(const URI &Item, const Cookie &CookieRec);
+ void AddCookie(const Cookie &CookieRec);
+ bool DoRealWrite();
+ bool Lock();
+ void UnLock();
+
+ public:
+ void Configure();
+ bool ReadCookies(bool DoLock = true);
+ bool WriteCookies() { if (IsEnabled && IsModified) DoRealWrite();
return true; };
+
+ bool GetCookies(const URI &Srv, string &Data);
+ bool SetCookie(const URI &Item, const string &Data);
+
+ CookieJar();
+ ~CookieJar() { WriteCookies(); };
+};
+
class CircleBuf
{
unsigned char *Buf;
@@ -106,6 +147,8 @@
int ServerFd;
URI ServerName;
+ CookieJar &Cookies;
+
bool HeaderLine(string Line);
bool Comp(URI Other) {return Other.Host == ServerName.Host &&
Other.Port == ServerName.Port;};
void Reset() {Major = 0; Minor = 0; Result = 0; Size = 0; StartPos =
0;
@@ -117,7 +160,7 @@
bool Open();
bool Close();
- ServerState(URI Srv,HttpMethod *Owner);
+ ServerState(URI Srv,HttpMethod *Owner,CookieJar &Cookies_);
~ServerState() {Close();};
};
@@ -137,6 +180,8 @@
static int FailFd;
static time_t FailTime;
static void SigTerm(int);
+
+ CookieJar Cookies;
public:
friend class ServerState;
Reply to: