43#include "InternalErr.h"
44#include "ResponseTooBigErr.h"
46#include "SignalHandler.h"
48#include "HTTPCacheInterruptHandler.h"
49#include "HTTPCacheMacros.h"
50#include "HTTPCacheTable.h"
59#define MKDIR(a, b) _mkdir((a))
62 int s = remove((a)); \
64 throw InternalErr(__FILE__, __LINE__, "Cache error; could not remove file: " + long_to_string(s)); \
66#define MKSTEMP(a) _open(_mktemp((a)), _O_CREAT, _S_IREAD | _S_IWRITE)
67#define DIR_SEPARATOR_CHAR '\\'
68#define DIR_SEPARATOR_STR "\\"
70#define MKDIR(a, b) mkdir((a), (b))
71#define MKSTEMP(a) mkstemp((a))
72#define DIR_SEPARATOR_CHAR '/'
73#define DIR_SEPARATOR_STR "/"
76#define CACHE_META ".meta"
77#define CACHE_INDEX ".index"
78#define CACHE_EMPTY_ETAG "@cache@"
80#define NO_LM_EXPIRATION 24 * 3600
81#define MAX_LM_EXPIRATION 48 * 3600
86#define LM_EXPIRATION(t) (min((MAX_LM_EXPIRATION), static_cast<int>((t) / 10)))
89const int CACHE_TABLE_SIZE = 1499;
101 for (
const char *ptr = url.c_str(); *ptr; ptr++)
102 hash = (int)((hash * 3 + (*(
unsigned char *)ptr)) % CACHE_TABLE_SIZE);
107HTTPCacheTable::HTTPCacheTable(
const string &cache_root,
int block_size)
108 : d_cache_root(cache_root), d_block_size(block_size), d_current_size(0), d_new_entries(0) {
109 d_cache_index = cache_root + CACHE_INDEX;
111 d_cache_table =
new CacheEntries *[CACHE_TABLE_SIZE];
114 for (
int i = 0; i < CACHE_TABLE_SIZE; ++i)
115 d_cache_table[i] = 0;
123static inline void delete_cache_entry(HTTPCacheTable::CacheEntry *e) {
124 DBG2(cerr <<
"Deleting CacheEntry: " << e << endl);
128HTTPCacheTable::~HTTPCacheTable() {
129 for (
int i = 0; i < CACHE_TABLE_SIZE; ++i) {
130 HTTPCacheTable::CacheEntries *cp = get_cache_table()[i];
133 for_each(cp->begin(), cp->end(), delete_cache_entry);
136 delete get_cache_table()[i];
137 get_cache_table()[i] = 0;
141 delete[] d_cache_table;
151class DeleteExpired :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
153 HTTPCacheTable &d_table;
156 DeleteExpired(HTTPCacheTable &table, time_t t) : d_time(t), d_table(table) {
161 void operator()(HTTPCacheTable::CacheEntry *&e) {
162 if (e && !e->readers && (e->freshness_lifetime < (e->corrected_initial_age + (d_time - e->response_time)))) {
163 DBG(cerr <<
"Deleting expired cache entry: " << e->url << endl);
164 d_table.remove_cache_entry(e);
172void HTTPCacheTable::delete_expired_entries(time_t time) {
174 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
175 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
177 for_each(slot->begin(), slot->end(), DeleteExpired(*
this, time));
178 slot->erase(remove(slot->begin(), slot->end(),
static_cast<HTTPCacheTable::CacheEntry *
>(0)), slot->end());
189class DeleteByHits :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
190 HTTPCacheTable &d_table;
194 DeleteByHits(HTTPCacheTable &table,
int hits) : d_table(table), d_hits(hits) {}
196 void operator()(HTTPCacheTable::CacheEntry *&e) {
197 if (e && !e->readers && e->hits <= d_hits) {
198 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
199 d_table.remove_cache_entry(e);
206void HTTPCacheTable::delete_by_hits(
int hits) {
207 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
208 if (get_cache_table()[cnt]) {
209 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
210 for_each(slot->begin(), slot->end(), DeleteByHits(*
this, hits));
211 slot->erase(remove(slot->begin(), slot->end(),
static_cast<HTTPCacheTable::CacheEntry *
>(0)), slot->end());
220class DeleteBySize :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
221 HTTPCacheTable &d_table;
225 DeleteBySize(HTTPCacheTable &table,
unsigned int size) : d_table(table), d_size(size) {}
227 void operator()(HTTPCacheTable::CacheEntry *&e) {
228 if (e && !e->readers && e->size > d_size) {
229 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
230 d_table.remove_cache_entry(e);
237void HTTPCacheTable::delete_by_size(
unsigned int size) {
238 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
239 if (get_cache_table()[cnt]) {
240 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
241 for_each(slot->begin(), slot->end(), DeleteBySize(*
this, size));
242 slot->erase(remove(slot->begin(), slot->end(),
static_cast<HTTPCacheTable::CacheEntry *
>(0)), slot->end());
263 return (REMOVE_BOOL(d_cache_index.c_str()) == 0);
275 FILE *fp = fopen(d_cache_index.c_str(),
"r");
283 while (!feof(fp) && fgets(line, 1024, fp)) {
285 DBG2(cerr << line << endl);
288 int res = fclose(fp);
290 DBG(cerr <<
"HTTPCache::cache_index_read - Failed to close " << (
void *)fp << endl);
308 istringstream iss(line);
310 iss >> entry->cachename;
313 if (entry->etag == CACHE_EMPTY_ETAG)
317 iss >> entry->expires;
323 iss >> entry->freshness_lifetime;
324 iss >> entry->response_time;
325 iss >> entry->corrected_initial_age;
327 iss >> entry->must_revalidate;
334class WriteOneCacheEntry :
public unary_function<HTTPCacheTable::CacheEntry *, void> {
339 WriteOneCacheEntry(FILE *fp) : d_fp(fp) {}
341 void operator()(HTTPCacheTable::CacheEntry *e) {
342 if (e && fprintf(d_fp,
"%s %s %s %ld %ld %ld %c %d %d %ld %ld %ld %c\r\n", e->url.c_str(), e->cachename.c_str(),
343 e->etag ==
"" ? CACHE_EMPTY_ETAG : e->etag.c_str(), (
long)(e->lm), (
long)(e->expires), e->size,
344 e->range ?
'1' :
'0',
345 e->hash, e->hits, (
long)(e->freshness_lifetime), (
long)(e->response_time),
346 (
long)(e->corrected_initial_age), e->must_revalidate ?
'1' :
'0') < 0)
347 throw Error(internal_error,
"Cache Index. Error writing cache index\n");
361 DBG(cerr <<
"Cache Index. Writing index " << d_cache_index << endl);
365 if ((fp = fopen(d_cache_index.c_str(),
"wb")) == NULL) {
366 throw Error(
string(
"Cache Index. Can't open `") + d_cache_index +
string(
"' for writing"));
372 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
373 HTTPCacheTable::CacheEntries *cp = get_cache_table()[cnt];
375 for_each(cp->begin(), cp->end(), WriteOneCacheEntry(fp));
379 int res = fclose(fp);
381 DBG(cerr <<
"HTTPCache::cache_index_write - Failed to close " << (
void *)fp << endl);
403 path << d_cache_root << hash;
406 mode_t mask = umask(0);
410 if (mkdir(path.str().c_str(), 0777) < 0 && errno != EEXIST) {
412 throw Error(internal_error,
413 "Could not create the directory for the cache at '" + path.str() +
"' (" + strerror(errno) +
").");
439 hash_dir +=
"\\dodsXXXXXX";
441 hash_dir +=
"/dodsXXXXXX";
446 vector<char> templat(hash_dir.size() + 1);
447 strncpy(templat.data(), hash_dir.c_str(), hash_dir.size() + 1);
457 int fd = MKSTEMP(templat.data());
461 throw Error(internal_error,
462 "The HTTP Cache could not create a file to hold the response; it will not be cached.");
465 entry->cachename = templat.data();
471static inline int entry_disk_space(
int size,
unsigned int block_size) {
472 unsigned int num_of_blocks = (size + block_size) / block_size;
474 DBG(cerr <<
"size: " << size <<
", block_size: " << block_size <<
", num_of_blocks: " << num_of_blocks << endl);
476 return num_of_blocks * block_size;
489 int hash = entry->hash;
490 if (hash > CACHE_TABLE_SIZE - 1 || hash < 0)
491 throw InternalErr(__FILE__, __LINE__,
"Hash value too large!");
493 if (!d_cache_table[hash])
494 d_cache_table[hash] =
new CacheEntries;
496 d_cache_table[hash]->push_back(entry);
498 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size <<
", entry->size: " << entry->size
499 <<
", block size: " << d_block_size << endl);
501 d_current_size += entry_disk_space(entry->size, d_block_size);
503 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size << endl);
505 increment_new_entries();
513 return get_locked_entry_from_cache_table(
get_hash(url), url);
525 DBG(cerr <<
"url: " << url <<
"; hash: " << hash << endl);
526 DBG(cerr <<
"d_cache_table: " << hex << d_cache_table << dec << endl);
527 if (d_cache_table[hash]) {
528 CacheEntries *cp = d_cache_table[hash];
529 for (CacheEntriesIter i = cp->begin(); i != cp->end(); ++i) {
532 if ((*i) && (*i)->url == url) {
533 (*i)->lock_read_response();
550 if (d_cache_table[hash]) {
551 CacheEntries *cp = d_cache_table[hash];
552 for (CacheEntriesIter i = cp->begin(); i != cp->end(); ++i) {
555 if ((*i) && (*i)->url == url) {
556 (*i)->lock_write_response();
576 throw InternalErr(__FILE__, __LINE__,
"Tried to delete a cache entry that is in use.");
578 REMOVE(entry->cachename.c_str());
579 REMOVE(
string(entry->cachename + CACHE_META).c_str());
581 DBG(cerr <<
"remove_cache_entry, current_size: " << get_current_size() << endl);
583 unsigned int eds = entry_disk_space(entry->size, get_block_size());
584 set_current_size((eds > get_current_size()) ? 0 : get_current_size() - eds);
586 DBG(cerr <<
"remove_cache_entry, current_size: " << get_current_size() << endl);
591class DeleteCacheEntry :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
596 DeleteCacheEntry(
HTTPCacheTable *c,
const string &url) : d_url(url), d_cache_table(c) {}
598 void operator()(HTTPCacheTable::CacheEntry *&e) {
599 if (e && e->url == d_url) {
600 e->lock_write_response();
602 e->unlock_write_response();
617 if (d_cache_table[hash]) {
618 CacheEntries *cp = d_cache_table[hash];
619 for_each(cp->begin(), cp->end(), DeleteCacheEntry(
this, url));
626class DeleteUnlockedCacheEntry :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
631 void operator()(HTTPCacheTable::CacheEntry *&e) {
640void HTTPCacheTable::delete_all_entries() {
643 for (
int cnt = 0; cnt < CACHE_TABLE_SIZE; cnt++) {
644 HTTPCacheTable::CacheEntries *slot = get_cache_table()[cnt];
646 for_each(slot->begin(), slot->end(), DeleteUnlockedCacheEntry(*
this));
647 slot->erase(remove(slot->begin(), slot->end(),
static_cast<HTTPCacheTable::CacheEntry *
>(0)), slot->end());
651 cache_index_delete();
668 entry->response_time = time(NULL);
669 time_t apparent_age = max(0,
static_cast<int>(entry->response_time - entry->date));
670 time_t corrected_received_age = max(apparent_age, entry->age);
671 time_t response_delay = entry->response_time - request_time;
672 entry->corrected_initial_age = corrected_received_age + response_delay;
677 time_t freshness_lifetime = entry->max_age;
678 if (freshness_lifetime < 0) {
679 if (entry->expires < 0) {
681 freshness_lifetime = default_expiration;
683 freshness_lifetime = LM_EXPIRATION(entry->date - entry->lm);
686 freshness_lifetime = entry->expires - entry->date;
689 entry->freshness_lifetime = max(0,
static_cast<int>(freshness_lifetime));
691 DBG2(cerr <<
"Cache....... Received Age " << entry->age <<
", corrected " << entry->corrected_initial_age
692 <<
", freshness lifetime " << entry->freshness_lifetime << endl);
707 const vector<string> &headers) {
709 for (i = headers.begin(); i != headers.end(); ++i) {
714 string::size_type colon = (*i).find(
':');
717 if (colon == string::npos)
720 string header = (*i).substr(0, (*i).find(
':'));
721 string value = (*i).substr((*i).find(
": ") + 2);
722 DBG2(cerr <<
"Header: " << header << endl);
723 DBG2(cerr <<
"Value: " << value << endl);
725 if (header ==
"ETag") {
727 }
else if (header ==
"Last-Modified") {
729 }
else if (header ==
"Expires") {
731 }
else if (header ==
"Date") {
733 }
else if (header ==
"Age") {
735 }
else if (header ==
"Content-Length") {
736 unsigned long clength = strtoul(value.c_str(), 0, 0);
737 if (clength > max_entry_size)
738 entry->set_no_cache(
true);
739 }
else if (header ==
"Cache-Control") {
743 if (value ==
"no-cache" || value ==
"no-store")
747 entry->set_no_cache(
true);
748 else if (value ==
"must-revalidate")
749 entry->must_revalidate =
true;
750 else if (value.find(
"max-age") != string::npos) {
751 string max_age = value.substr(value.find(
"=") + 1);
763 d_locked_entries[body] = entry;
766void HTTPCacheTable::uncouple_entry_from_data(FILE *body) {
768 HTTPCacheTable::CacheEntry *entry = d_locked_entries[body];
770 throw InternalErr(
"There is no cache entry for the response given.");
772 d_locked_entries.erase(body);
773 entry->unlock_read_response();
775 if (entry->readers < 0)
776 throw InternalErr(
"An unlocked entry was released");
779bool HTTPCacheTable::is_locked_read_responses() {
return !d_locked_entries.empty(); }
A class for error processing.
void create_location(CacheEntry *entry)
void calculate_time(HTTPCacheTable::CacheEntry *entry, int default_expiration, time_t request_time)
string create_hash_directory(int hash)
bool cache_index_delete()
CacheEntry * cache_index_parse_line(const char *line)
void parse_headers(HTTPCacheTable::CacheEntry *entry, unsigned long max_entry_size, const vector< string > &headers)
CacheEntry * get_write_locked_entry_from_cache_table(const string &url)
void remove_cache_entry(HTTPCacheTable::CacheEntry *entry)
void add_entry_to_cache_table(CacheEntry *entry)
void remove_entry_from_cache_table(const string &url)
A class for software fault reporting.
top level DAP object to house generic methods
int get_hash(const string &url)
time_t parse_time(const char *str, bool expand)