48#include "InternalErr.h"
49#include "ResponseTooBigErr.h"
51#include "SignalHandler.h"
53#include "HTTPCacheInterruptHandler.h"
54#include "HTTPCacheTable.h"
55#include "HTTPCacheMacros.h"
64#define MKDIR(a,b) _mkdir((a))
65#define REMOVE(a) do { \
66 int s = remove((a)); \
68 throw InternalErr(__FILE__, __LINE__, "Cache error; could not remove file: " + long_to_string(s)); \
70#define MKSTEMP(a) _open(_mktemp((a)),_O_CREAT,_S_IREAD|_S_IWRITE)
71#define DIR_SEPARATOR_CHAR '\\'
72#define DIR_SEPARATOR_STR "\\"
74#define MKDIR(a,b) mkdir((a), (b))
75#define MKSTEMP(a) mkstemp((a))
76#define DIR_SEPARATOR_CHAR '/'
77#define DIR_SEPARATOR_STR "/"
80#define CACHE_META ".meta"
81#define CACHE_INDEX ".index"
82#define CACHE_EMPTY_ETAG "@cache@"
84#define NO_LM_EXPIRATION 24*3600
85#define MAX_LM_EXPIRATION 48*3600
90#define LM_EXPIRATION(t) (min((MAX_LM_EXPIRATION), static_cast<int>((t) / 10)))
93const int CACHE_TABLE_SIZE = 1499;
107 for (
const char *ptr = url.c_str(); *ptr; ptr++)
108 hash = (int)((hash * 3 + (*(
unsigned char *)ptr)) % CACHE_TABLE_SIZE);
113HTTPCacheTable::HTTPCacheTable(
const string &cache_root,
int block_size) :
114 d_cache_root(cache_root), d_block_size(block_size), d_current_size(0), d_new_entries(0)
116 d_cache_index = cache_root + CACHE_INDEX;
118 d_cache_table =
new CacheEntries*[CACHE_TABLE_SIZE];
121 for (
int i = 0; i < CACHE_TABLE_SIZE; ++i)
122 d_cache_table[i] = 0;
131delete_cache_entry(HTTPCacheTable::CacheEntry *e)
133 DBG2(cerr <<
"Deleting CacheEntry: " << e << endl);
137HTTPCacheTable::~HTTPCacheTable()
139 for (
int i = 0; i < CACHE_TABLE_SIZE; ++i) {
140 HTTPCacheTable::CacheEntries *cp = get_cache_table()[i];
143 for_each(cp->begin(), cp->end(), delete_cache_entry);
146 delete get_cache_table()[i];
147 get_cache_table()[i] = 0;
151 delete[] d_cache_table;
161class DeleteExpired :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
163 HTTPCacheTable &d_table;
166 DeleteExpired(HTTPCacheTable &table, time_t t) :
167 d_time(t), d_table(table) {
172 void operator()(HTTPCacheTable::CacheEntry *&e) {
173 if (e && !e->readers && (e->freshness_lifetime
174 < (e->corrected_initial_age + (d_time - e->response_time)))) {
175 DBG(cerr <<
"Deleting expired cache entry: " << e->url << endl);
176 d_table.remove_cache_entry(e);
183void HTTPCacheTable::delete_expired_entries(time_t time) {
185 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
186 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
188 for_each(slot->begin(), slot->end(), DeleteExpired(*
this, time));
189 slot->erase(remove(slot->begin(), slot->end(),
190 static_cast<HTTPCacheTable::CacheEntry *
>(0)), slot->end());
201class DeleteByHits :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
202 HTTPCacheTable &d_table;
206 DeleteByHits(HTTPCacheTable &table,
int hits) :
207 d_table(table), d_hits(hits) {
210 void operator()(HTTPCacheTable::CacheEntry *&e) {
211 if (e && !e->readers && e->hits <= d_hits) {
212 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
213 d_table.remove_cache_entry(e);
220HTTPCacheTable::delete_by_hits(
int hits) {
221 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
222 if (get_cache_table()[cnt]) {
223 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
224 for_each(slot->begin(), slot->end(), DeleteByHits(*
this, hits));
225 slot->erase(remove(slot->begin(), slot->end(),
226 static_cast<HTTPCacheTable::CacheEntry*
>(0)),
237class DeleteBySize :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
238 HTTPCacheTable &d_table;
242 DeleteBySize(HTTPCacheTable &table,
unsigned int size) :
243 d_table(table), d_size(size) {
246 void operator()(HTTPCacheTable::CacheEntry *&e) {
247 if (e && !e->readers && e->size > d_size) {
248 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
249 d_table.remove_cache_entry(e);
255void HTTPCacheTable::delete_by_size(
unsigned int size) {
256 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
257 if (get_cache_table()[cnt]) {
258 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
259 for_each(slot->begin(), slot->end(), DeleteBySize(*
this, size));
260 slot->erase(remove(slot->begin(), slot->end(),
261 static_cast<HTTPCacheTable::CacheEntry*
>(0)),
282HTTPCacheTable::cache_index_delete()
286 return (REMOVE_BOOL(d_cache_index.c_str()) == 0);
298HTTPCacheTable::cache_index_read()
300 FILE *fp = fopen(d_cache_index.c_str(),
"r");
308 while (!feof(fp) && fgets(line, 1024, fp)) {
309 add_entry_to_cache_table(cache_index_parse_line(line));
310 DBG2(cerr << line << endl);
313 int res = fclose(fp) ;
315 DBG(cerr <<
"HTTPCache::cache_index_read - Failed to close " << (
void *)fp << endl);
331HTTPCacheTable::cache_index_parse_line(
const char *line)
335 istringstream iss(line);
337 iss >> entry->cachename;
340 if (entry->etag == CACHE_EMPTY_ETAG)
344 iss >> entry->expires;
350 iss >> entry->freshness_lifetime;
351 iss >> entry->response_time;
352 iss >> entry->corrected_initial_age;
354 iss >> entry->must_revalidate;
361class WriteOneCacheEntry :
362 public unary_function<HTTPCacheTable::CacheEntry *, void>
368 WriteOneCacheEntry(FILE *fp) : d_fp(fp)
371 void operator()(HTTPCacheTable::CacheEntry *e)
373 if (e && fprintf(d_fp,
374 "%s %s %s %ld %ld %ld %c %d %d %ld %ld %ld %c\r\n",
376 e->cachename.c_str(),
377 e->etag ==
"" ? CACHE_EMPTY_ETAG : e->etag.c_str(),
381 e->range ?
'1' :
'0',
384 (
long)(e->freshness_lifetime),
385 (
long)(e->response_time),
386 (
long)(e->corrected_initial_age),
387 e->must_revalidate ?
'1' :
'0') < 0)
388 throw Error(internal_error,
"Cache Index. Error writing cache index\n");
402HTTPCacheTable::cache_index_write()
404 DBG(cerr <<
"Cache Index. Writing index " << d_cache_index << endl);
408 if ((fp = fopen(d_cache_index.c_str(),
"wb")) == NULL) {
409 throw Error(
string(
"Cache Index. Can't open `") + d_cache_index
410 +
string(
"' for writing"));
416 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
417 HTTPCacheTable::CacheEntries *cp = get_cache_table()[cnt];
419 for_each(cp->begin(), cp->end(), WriteOneCacheEntry(fp));
423 int res = fclose(fp);
425 DBG(cerr <<
"HTTPCache::cache_index_write - Failed to close "
426 << (
void *)fp << endl);
447HTTPCacheTable::create_hash_directory(
int hash)
450 struct stat stat_info;
453 path << d_cache_root << hash;
454 string p = path.str();
456 if (stat(p.c_str(), &stat_info) == -1) {
457 DBG2(cerr <<
"Cache....... Create dir " << p << endl);
458 if (MKDIR(p.c_str(), 0777) < 0) {
459 DBG2(cerr <<
"Cache....... Can't create..." << endl);
460 throw Error(
"Could not create cache slot to hold response! Check the write permissions on your disk cache directory. Cache root: " + d_cache_root +
".");
464 DBG2(cerr <<
"Cache....... Directory " << p <<
" already exists"
472 path << d_cache_root << hash;
475 mode_t mask = umask(0);
479 if (mkdir(path.str().c_str(), 0777) < 0 && errno != EEXIST) {
481 throw Error(internal_error,
"Could not create the directory for the cache at '" + path.str() +
"' (" + strerror(errno) +
").");
507 string hash_dir = create_hash_directory(entry->hash);
509 hash_dir +=
"\\dodsXXXXXX";
511 hash_dir +=
"/dodsXXXXXX";
516 vector<char> templat(hash_dir.size() + 1);
517 strncpy(templat.data(), hash_dir.c_str(), hash_dir.size() + 1);
527 int fd = MKSTEMP(templat.data());
531 throw Error(internal_error,
"The HTTP Cache could not create a file to hold the response; it will not be cached.");
534 entry->cachename = templat.data();
542entry_disk_space(
int size,
unsigned int block_size)
544 unsigned int num_of_blocks = (size + block_size) / block_size;
546 DBG(cerr <<
"size: " << size <<
", block_size: " << block_size
547 <<
", num_of_blocks: " << num_of_blocks << endl);
549 return num_of_blocks * block_size;
564 int hash = entry->hash;
565 if (hash > CACHE_TABLE_SIZE-1 || hash < 0)
566 throw InternalErr(__FILE__, __LINE__,
"Hash value too large!");
568 if (!d_cache_table[hash])
569 d_cache_table[hash] =
new CacheEntries;
571 d_cache_table[hash]->push_back(entry);
573 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size
574 <<
", entry->size: " << entry->size <<
", block size: " << d_block_size
577 d_current_size += entry_disk_space(entry->size, d_block_size);
579 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size << endl);
581 increment_new_entries();
588HTTPCacheTable::get_locked_entry_from_cache_table(
const string &url)
590 return get_locked_entry_from_cache_table(
get_hash(url), url);
601HTTPCacheTable::get_locked_entry_from_cache_table(
int hash,
const string &url)
603 DBG(cerr <<
"url: " << url <<
"; hash: " << hash << endl);
604 DBG(cerr <<
"d_cache_table: " << hex << d_cache_table << dec << endl);
605 if (d_cache_table[hash]) {
606 CacheEntries *cp = d_cache_table[hash];
607 for (CacheEntriesIter i = cp->begin(); i != cp->end(); ++i) {
610 if ((*i) && (*i)->url == url) {
611 (*i)->lock_read_response();
626HTTPCacheTable::CacheEntry *
627HTTPCacheTable::get_write_locked_entry_from_cache_table(
const string &url)
630 if (d_cache_table[hash]) {
631 CacheEntries *cp = d_cache_table[hash];
632 for (CacheEntriesIter i = cp->begin(); i != cp->end(); ++i) {
635 if ((*i) && (*i)->url == url) {
636 (*i)->lock_write_response();
658 throw InternalErr(__FILE__, __LINE__,
"Tried to delete a cache entry that is in use.");
660 REMOVE(entry->cachename.c_str());
661 REMOVE(
string(entry->cachename + CACHE_META).c_str());
663 DBG(cerr <<
"remove_cache_entry, current_size: " << get_current_size() << endl);
665 unsigned int eds = entry_disk_space(entry->size, get_block_size());
666 set_current_size((eds > get_current_size()) ? 0 : get_current_size() - eds);
668 DBG(cerr <<
"remove_cache_entry, current_size: " << get_current_size() << endl);
673class DeleteCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void>
680 : d_url(url), d_cache_table(c)
683 void operator()(HTTPCacheTable::CacheEntry *&e)
685 if (e && e->url == d_url) {
686 e->lock_write_response();
688 e->unlock_write_response();
701HTTPCacheTable::remove_entry_from_cache_table(
const string &url)
704 if (d_cache_table[hash]) {
705 CacheEntries *cp = d_cache_table[hash];
706 for_each(cp->begin(), cp->end(), DeleteCacheEntry(
this, url));
714class DeleteUnlockedCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
722 void operator()(HTTPCacheTable::CacheEntry *&e)
732void HTTPCacheTable::delete_all_entries()
736 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
737 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
739 for_each(slot->begin(), slot->end(), DeleteUnlockedCacheEntry(*
this));
740 slot->erase(remove(slot->begin(), slot->end(),
static_cast<HTTPCacheTable::CacheEntry *
> (0)), slot->end());
744 cache_index_delete();
763 entry->response_time = time(NULL);
764 time_t apparent_age = max(0,
static_cast<int>(entry->response_time - entry->date));
765 time_t corrected_received_age = max(apparent_age, entry->age);
766 time_t response_delay = entry->response_time - request_time;
767 entry->corrected_initial_age = corrected_received_age + response_delay;
772 time_t freshness_lifetime = entry->max_age;
773 if (freshness_lifetime < 0) {
774 if (entry->expires < 0) {
776 freshness_lifetime = default_expiration;
779 freshness_lifetime = LM_EXPIRATION(entry->date - entry->lm);
783 freshness_lifetime = entry->expires - entry->date;
786 entry->freshness_lifetime = max(0,
static_cast<int>(freshness_lifetime));
788 DBG2(cerr <<
"Cache....... Received Age " << entry->age
789 <<
", corrected " << entry->corrected_initial_age
790 <<
", freshness lifetime " << entry->freshness_lifetime << endl);
805 const vector<string> &headers)
807 vector<string>::const_iterator i;
808 for (i = headers.begin(); i != headers.end(); ++i) {
813 string::size_type colon = (*i).find(
':');
816 if (colon == string::npos)
819 string header = (*i).substr(0, (*i).find(
':'));
820 string value = (*i).substr((*i).find(
": ") + 2);
821 DBG2(cerr <<
"Header: " << header << endl);DBG2(cerr <<
"Value: " << value << endl);
823 if (header ==
"ETag") {
826 else if (header ==
"Last-Modified") {
829 else if (header ==
"Expires") {
832 else if (header ==
"Date") {
835 else if (header ==
"Age") {
838 else if (header ==
"Content-Length") {
839 unsigned long clength = strtoul(value.c_str(), 0, 0);
840 if (clength > max_entry_size)
841 entry->set_no_cache(
true);
843 else if (header ==
"Cache-Control") {
847 if (value ==
"no-cache" || value ==
"no-store")
851 entry->set_no_cache(
true);
852 else if (value ==
"must-revalidate")
853 entry->must_revalidate =
true;
854 else if (value.find(
"max-age") != string::npos) {
855 string max_age = value.substr(value.find(
"=") + 1);
867 d_locked_entries[body] = entry;
870void HTTPCacheTable::uncouple_entry_from_data(FILE *body) {
872 HTTPCacheTable::CacheEntry *entry = d_locked_entries[body];
874 throw InternalErr(
"There is no cache entry for the response given.");
876 d_locked_entries.erase(body);
877 entry->unlock_read_response();
879 if (entry->readers < 0)
880 throw InternalErr(
"An unlocked entry was released");
883bool HTTPCacheTable::is_locked_read_responses() {
884 return !d_locked_entries.empty();
A class for error processing.
void remove_cache_entry(HTTPCacheTable::CacheEntry *entry)
A class for software fault reporting.
top level DAP object to house generic methods
int get_hash(const string &url)
time_t parse_time(const char *str, bool expand)