/* * $Id: url.c,v 1.144.2.1 2007/03/03 18:34:13 hno Exp $ * * DEBUG: section 23 URL Parsing * AUTHOR: Duane Wessels * * SQUID Web Proxy Cache http://www.squid-cache.org/ * ---------------------------------------------------------- * * Squid is the result of efforts by numerous individuals from * the Internet community; see the CONTRIBUTORS file for full * details. Many organizations have provided support for Squid's * development; see the SPONSORS file for full details. Squid is * Copyrighted (C) 2001 by the Regents of the University of * California; see the COPYRIGHT file for full details. Squid * incorporates software developed and/or copyrighted by other * sources; see the CREDITS file for full details. * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. * */ #include "squid.h" const char *RequestMethodStr[] = { "NONE", "GET", "POST", "PUT", "HEAD", "CONNECT", "TRACE", "PURGE", "OPTIONS", "DELETE", "PROPFIND", "PROPPATCH", "MKCOL", "COPY", "MOVE", "LOCK", "UNLOCK", "BMOVE", "BDELETE", "BPROPFIND", "BPROPPATCH", "BCOPY", "SEARCH", "SUBSCRIBE", "UNSUBSCRIBE", "POLL", "REPORT", "MKACTIVITY", "CHECKOUT", "MERGE", "%EXT00", "%EXT01", "%EXT02", "%EXT03", "%EXT04", "%EXT05", "%EXT06", "%EXT07", "%EXT08", "%EXT09", "%EXT10", "%EXT11", "%EXT12", "%EXT13", "%EXT14", "%EXT15", "%EXT16", "%EXT17", "%EXT18", "%EXT19", "ERROR" }; const char *ProtocolStr[] = { "NONE", "http", "ftp", "gopher", "wais", "cache_object", "icp", #if USE_HTCP "htcp", #endif "urn", "whois", "internal", "https", "TOTAL" }; static request_t *urnParse(method_t method, char *urn); static const char valid_hostname_chars_u[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789-._"; static const char valid_hostname_chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "abcdefghijklmnopqrstuvwxyz" "0123456789-."; /* convert %xx in url string to a character * Allocate a new string and return a pointer to converted string */ char * url_convert_hex(char *org_url, int allocate) { static char code[] = "00"; char *url = NULL; char *s = NULL; char *t = NULL; url = allocate ? (char *) xstrdup(org_url) : org_url; if ((int) strlen(url) < 3 || !strchr(url, '%')) return url; for (s = t = url; *s; s++) { if (*s == '%' && *(s + 1) && *(s + 2)) { code[0] = *(++s); code[1] = *(++s); *t++ = (char) strtol(code, NULL, 16); } else { *t++ = *s; } } do { *t++ = *s; } while (*s++); return url; } void urlInitialize(void) { debug(23, 5) ("urlInitialize: Initializing...\n"); assert(sizeof(ProtocolStr) == (PROTO_MAX + 1) * sizeof(char *)); memset(&null_request_flags, '\0', sizeof(null_request_flags)); /* * These test that our matchDomainName() function works the * way we expect it to. */ assert(0 == matchDomainName("foo.com", "foo.com")); assert(0 == matchDomainName(".foo.com", "foo.com")); assert(0 == matchDomainName("foo.com", ".foo.com")); assert(0 == matchDomainName(".foo.com", ".foo.com")); assert(0 == matchDomainName("x.foo.com", ".foo.com")); assert(0 != matchDomainName("x.foo.com", "foo.com")); assert(0 != matchDomainName("foo.com", "x.foo.com")); assert(0 != matchDomainName("bar.com", "foo.com")); assert(0 != matchDomainName(".bar.com", "foo.com")); assert(0 != matchDomainName(".bar.com", ".foo.com")); assert(0 != matchDomainName("bar.com", ".foo.com")); assert(0 < matchDomainName("zzz.com", "foo.com")); assert(0 > matchDomainName("aaa.com", "foo.com")); assert(0 == matchDomainName("FOO.com", "foo.COM")); assert(0 < matchDomainName("bfoo.com", "afoo.com")); assert(0 > matchDomainName("afoo.com", "bfoo.com")); assert(0 < matchDomainName("x-foo.com", ".foo.com")); /* more cases? */ } method_t urlParseMethod(const char *s) { method_t method = METHOD_NONE; /* * This check for '%' makes sure that we don't * match one of the extension method placeholders, * which have the form %EXT[0-9][0-9] */ if (*s == '%') return METHOD_NONE; for (method++; method < METHOD_ENUM_END; method++) { if (0 == strcasecmp(s, RequestMethodStr[method])) return method; } return METHOD_NONE; } protocol_t urlParseProtocol(const char *s) { /* test common stuff first */ if (strcasecmp(s, "http") == 0) return PROTO_HTTP; if (strcasecmp(s, "ftp") == 0) return PROTO_FTP; if (strcasecmp(s, "https") == 0) return PROTO_HTTPS; if (strcasecmp(s, "file") == 0) return PROTO_FTP; if (strcasecmp(s, "gopher") == 0) return PROTO_GOPHER; if (strcasecmp(s, "wais") == 0) return PROTO_WAIS; if (strcasecmp(s, "cache_object") == 0) return PROTO_CACHEOBJ; if (strcasecmp(s, "urn") == 0) return PROTO_URN; if (strcasecmp(s, "whois") == 0) return PROTO_WHOIS; if (strcasecmp(s, "internal") == 0) return PROTO_INTERNAL; return PROTO_NONE; } int urlDefaultPort(protocol_t p) { switch (p) { case PROTO_HTTP: return 80; case PROTO_HTTPS: return 443; case PROTO_FTP: return 21; case PROTO_GOPHER: return 70; case PROTO_WAIS: return 210; case PROTO_CACHEOBJ: case PROTO_INTERNAL: return CACHE_HTTP_PORT; case PROTO_WHOIS: return 43; default: return 0; } } request_t * urlParse(method_t method, char *url) { LOCAL_ARRAY(char, proto, MAX_URL); LOCAL_ARRAY(char, login, MAX_URL); LOCAL_ARRAY(char, host, MAX_URL); LOCAL_ARRAY(char, urlpath, MAX_URL); request_t *request = NULL; char *t = NULL; char *q = NULL; int port; protocol_t protocol = PROTO_NONE; int l; proto[0] = host[0] = urlpath[0] = login[0] = '\0'; if ((l = strlen(url)) + Config.appendDomainLen > (MAX_URL - 1)) { /* terminate so it doesn't overflow other buffers */ *(url + (MAX_URL >> 1)) = '\0'; debug(23, 1) ("urlParse: URL too large (%d bytes)\n", l); return NULL; } if (method == METHOD_CONNECT) { port = CONNECT_PORT; if (sscanf(url, "%[^:]:%d", host, &port) < 1) return NULL; } else if (!strncmp(url, "urn:", 4)) { return urnParse(method, url); } else { if (sscanf(url, "%[^:]://%[^/]%[^\r\n]", proto, host, urlpath) < 2) return NULL; protocol = urlParseProtocol(proto); port = urlDefaultPort(protocol); /* Is there any login informaiton? */ if ((t = strrchr(host, '@'))) { strcpy((char *) login, (char *) host); t = strrchr(login, '@'); *t = 0; strcpy((char *) host, t + 1); } if ((t = strrchr(host, ':'))) { *t++ = '\0'; if (*t != '\0') port = atoi(t); } } for (t = host; *t; t++) *t = xtolower(*t); if (stringHasWhitespace(host)) { if (URI_WHITESPACE_STRIP == Config.uri_whitespace) { t = q = host; while (*t) { if (!xisspace(*t)) *q++ = *t; t++; } *q = '\0'; } } if (Config.onoff.check_hostnames && strspn(host, Config.onoff.allow_underscore ? valid_hostname_chars_u : valid_hostname_chars) != strlen(host)) { debug(23, 1) ("urlParse: Illegal character in hostname '%s'\n", host); return NULL; } if (Config.appendDomain && !strchr(host, '.')) strncat(host, Config.appendDomain, SQUIDHOSTNAMELEN - strlen(host) - 1); /* remove trailing dots from hostnames */ while ((l = strlen(host)) > 0 && host[--l] == '.') host[l] = '\0'; /* reject duplicate or leading dots */ if (strstr(host, "..") || *host == '.') { debug(23, 1) ("urlParse: Illegal hostname '%s'\n", host); return NULL; } if (port < 1 || port > 65535) { debug(23, 3) ("urlParse: Invalid port '%d'\n", port); return NULL; } #ifdef HARDCODE_DENY_PORTS /* These ports are filtered in the default squid.conf, but * maybe someone wants them hardcoded... */ if (port == 7 || port == 9 || port == 19) { debug(23, 0) ("urlParse: Deny access to port %d\n", port); return NULL; } #endif if (stringHasWhitespace(urlpath)) { debug(23, 2) ("urlParse: URI has whitespace: {%s}\n", url); switch (Config.uri_whitespace) { case URI_WHITESPACE_DENY: return NULL; case URI_WHITESPACE_ALLOW: break; case URI_WHITESPACE_ENCODE: t = rfc1738_escape_unescaped(urlpath); xstrncpy(urlpath, t, MAX_URL); break; case URI_WHITESPACE_CHOP: *(urlpath + strcspn(urlpath, w_space)) = '\0'; break; case URI_WHITESPACE_STRIP: default: t = q = urlpath; while (*t) { if (!xisspace(*t)) *q++ = *t; t++; } *q = '\0'; } } request = requestCreate(method, protocol, urlpath); xstrncpy(request->host, host, SQUIDHOSTNAMELEN); xstrncpy(request->login, login, MAX_LOGIN_SZ); request->port = (u_short) port; return request; } static request_t * urnParse(method_t method, char *urn) { debug(50, 5) ("urnParse: %s\n", urn); return requestCreate(method, PROTO_URN, urn + 4); } const char * urlCanonical(request_t * request) { LOCAL_ARRAY(char, portbuf, 32); LOCAL_ARRAY(char, urlbuf, MAX_URL); if (request->canonical) return request->canonical; if (request->protocol == PROTO_URN) { snprintf(urlbuf, MAX_URL, "urn:%s", strBuf(request->urlpath)); } else { switch (request->method) { case METHOD_CONNECT: snprintf(urlbuf, MAX_URL, "%s:%d", request->host, request->port); break; default: portbuf[0] = '\0'; if (request->port != urlDefaultPort(request->protocol)) snprintf(portbuf, 32, ":%d", request->port); snprintf(urlbuf, MAX_URL, "%s://%s%s%s%s%s", ProtocolStr[request->protocol], request->login, *request->login ? "@" : null_string, request->host, portbuf, strBuf(request->urlpath)); break; } } return (request->canonical = xstrdup(urlbuf)); } char * urlCanonicalClean(const request_t * request) { LOCAL_ARRAY(char, buf, MAX_URL); LOCAL_ARRAY(char, portbuf, 32); LOCAL_ARRAY(char, loginbuf, MAX_LOGIN_SZ + 1); char *t; if (request->protocol == PROTO_URN) { snprintf(buf, MAX_URL, "urn:%s", strBuf(request->urlpath)); } else { switch (request->method) { case METHOD_CONNECT: snprintf(buf, MAX_URL, "%s:%d", request->host, request->port); break; default: portbuf[0] = '\0'; if (request->port != urlDefaultPort(request->protocol)) snprintf(portbuf, 32, ":%d", request->port); loginbuf[0] = '\0'; if ((int) strlen(request->login) > 0) { strcpy(loginbuf, request->login); if ((t = strchr(loginbuf, ':'))) *t = '\0'; strcat(loginbuf, "@"); } snprintf(buf, MAX_URL, "%s://%s%s%s%s", ProtocolStr[request->protocol], loginbuf, request->host, portbuf, strBuf(request->urlpath)); /* * strip arguments AFTER a question-mark */ if (Config.onoff.strip_query_terms) if ((t = strchr(buf, '?'))) *(++t) = '\0'; break; } } if (stringHasCntl(buf)) xstrncpy(buf, rfc1738_escape_unescaped(buf), MAX_URL); return buf; } /* * matchDomainName() compares a hostname with a domainname according * to the following rules: * * HOST DOMAIN MATCH? * ------------- ------------- ------ * foo.com foo.com YES * .foo.com foo.com YES * x.foo.com foo.com NO * foo.com .foo.com YES * .foo.com .foo.com YES * x.foo.com .foo.com YES * * We strip leading dots on hosts (but not domains!) so that * ".foo.com" is is always the same as "foo.com". * * Return values: * 0 means the host matches the domain * 1 means the host is greater than the domain * -1 means the host is less than the domain */ int matchDomainName(const char *h, const char *d) { int dl; int hl; while ('.' == *h) h++; hl = strlen(h); dl = strlen(d); /* * Start at the ends of the two strings and work towards the * beginning. */ while (xtolower(h[--hl]) == xtolower(d[--dl])) { if (hl == 0 && dl == 0) { /* * We made it all the way to the beginning of both * strings without finding any difference. */ return 0; } if (0 == hl) { /* * The host string is shorter than the domain string. * There is only one case when this can be a match. * If the domain is just one character longer, and if * that character is a leading '.' then we call it a * match. */ if (1 == dl && '.' == d[0]) return 0; else return -1; } if (0 == dl) { /* * The domain string is shorter than the host string. * This is a match only if the first domain character * is a leading '.'. */ if ('.' == d[0]) return 0; else return 1; } } /* * We found different characters in the same position (from the end). */ /* * If one of those character is '.' then its special. In order * for splay tree sorting to work properly, "x-foo.com" must * be greater than ".foo.com" even though '-' is less than '.'. */ if ('.' == d[dl]) return 1; if ('.' == h[hl]) return -1; return (xtolower(h[hl]) - xtolower(d[dl])); } int urlCheckRequest(const request_t * r) { int rc = 0; /* protocol "independent" methods */ if (r->method == METHOD_CONNECT) return 1; if (r->method == METHOD_TRACE) return 1; if (r->method == METHOD_PURGE) return 1; /* does method match the protocol? */ switch (r->protocol) { case PROTO_URN: case PROTO_HTTP: case PROTO_INTERNAL: case PROTO_CACHEOBJ: rc = 1; break; case PROTO_FTP: if (r->method == METHOD_PUT) rc = 1; case PROTO_GOPHER: case PROTO_WAIS: case PROTO_WHOIS: if (r->method == METHOD_GET) rc = 1; else if (r->method == METHOD_HEAD) rc = 1; break; case PROTO_HTTPS: #ifdef USE_SSL rc = 1; break; #else /* * Squid can't originate an SSL connection, so it should * never receive an "https:" URL. It should always be * CONNECT instead. */ rc = 0; #endif default: break; } return rc; } /* * Quick-n-dirty host extraction from a URL. Steps: * Look for a colon * Skip any '/' after the colon * Copy the next SQUID_MAXHOSTNAMELEN bytes to host[] * Look for an ending '/' or ':' and terminate * Look for login info preceeded by '@' */ char * urlHostname(const char *url) { LOCAL_ARRAY(char, host, SQUIDHOSTNAMELEN); char *t; host[0] = '\0'; if (NULL == (t = strchr(url, ':'))) return NULL; t++; while (*t != '\0' && *t == '/') t++; xstrncpy(host, t, SQUIDHOSTNAMELEN); if ((t = strchr(host, '/'))) *t = '\0'; if ((t = strchr(host, ':'))) *t = '\0'; if ((t = strrchr(host, '@'))) { t++; xmemmove(host, t, strlen(t) + 1); } return host; } static void urlExtMethodAdd(const char *mstr) { method_t method = 0; for (method++; method < METHOD_ENUM_END; method++) { if (0 == strcmp(mstr, RequestMethodStr[method])) { debug(23, 2) ("Extension method '%s' already exists\n", mstr); return; } if (0 != strncmp("%EXT", RequestMethodStr[method], 4)) continue; /* Don't free statically allocated "%EXTnn" string */ if (0 == strncmp("%EXT_", RequestMethodStr[method], 5)) safe_free(RequestMethodStr[method]); RequestMethodStr[method] = xstrdup(mstr); debug(23, 1) ("Extension method '%s' added, enum=%d\n", mstr, (int) method); return; } debug(23, 1) ("WARNING: Could not add new extension method '%s' due to lack of array space\n", mstr); } void parse_extension_method(const char *(*_methods)[]) { char *token; char *t = strtok(NULL, ""); while ((token = strwordtok(NULL, &t))) { urlExtMethodAdd(token); } } void free_extension_method(const char *(*_methods)[]) { method_t method; char **methods = (char **) _methods; for (method = METHOD_EXT00; method < METHOD_ENUM_END; method++) { if (*methods[method] != '%') { char buf[32]; snprintf(buf, sizeof(buf), "%%EXT_%02d", method - METHOD_EXT00); safe_free(methods[method]); methods[method] = xstrdup(buf); } } } void dump_extension_method(StoreEntry * entry, const char *name, const char **methods) { method_t method; for (method = METHOD_EXT00; method < METHOD_ENUM_END; method++) { if (*methods[method] != '%') { storeAppendPrintf(entry, "%s %s\n", name, methods[method]); } } }