diff -Nru mapcache-1.2.1/apache/mod_mapcache.c mapcache-1.4.1/apache/mod_mapcache.c --- mapcache-1.2.1/apache/mod_mapcache.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/apache/mod_mapcache.c 2016-02-25 15:47:16.000000000 +0000 @@ -59,8 +59,6 @@ typedef struct mapcache_context_apache_request mapcache_context_apache_request; typedef struct mapcache_context_apache_server mapcache_context_apache_server; -apr_pool_t *pchild = NULL; - struct mapcache_context_apache { mapcache_context ctx; }; @@ -75,6 +73,19 @@ request_rec *request; }; +typedef struct mapcache_alias_entry mapcache_alias_entry; + +struct mapcache_alias_entry { + char *endpoint; + char *configfile; + mapcache_cfg *cfg; + mapcache_connection_pool *cp; +}; + +struct mapcache_server_cfg { + apr_array_header_t *aliases; /**< list of mapcache configurations aliased to a server uri */ +}; + void apache_context_server_log(mapcache_context *c, mapcache_log_level level, char *message, ...) { mapcache_context_apache_server *ctx = (mapcache_context_apache_server*)c; @@ -185,21 +196,42 @@ { mapcache_context_apache_request *ctx = apr_pcalloc(r->pool, sizeof(mapcache_context_apache_request)); mapcache_server_cfg *cfg = NULL; - mapcache_cfg *config = NULL; + const char *mapcache_alias; + mapcache_alias_entry *alias_entry; + int i; + mapcache_context *mctx = (mapcache_context*)ctx; - ctx->ctx.ctx.pool = r->pool; - ctx->ctx.ctx.process_pool = pchild; + mctx->pool = r->pool; #ifdef APR_HAS_THREADS - ctx->ctx.ctx.threadlock = thread_mutex; + mctx->threadlock = thread_mutex; #endif /* lookup the configuration object given the configuration file name */ cfg = ap_get_module_config(r->server->module_config, &mapcache_module); - config = apr_hash_get(cfg->aliases,(void*)r->filename,APR_HASH_KEY_STRING); - ctx->ctx.ctx.config = config; - ctx->request = r; - init_apache_request_context(ctx); - return ctx; + if(!cfg || !cfg->aliases) { + return NULL; + } + + mapcache_alias = apr_table_get(r->notes,"mapcache_alias_entry"); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "using mapcache config %s", mapcache_config_file); + if(!mapcache_alias) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "mapcache module bug? no mapcache_alias_entry found"); + return NULL; + } + + for(i=0; ialiases->nelts; i++) { + alias_entry = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + if(strcmp(alias_entry->endpoint,mapcache_alias)) + continue; + + mctx->config = alias_entry->cfg; + ctx->request = r; + mctx->connection_pool = alias_entry->cp; + init_apache_request_context(ctx); + return ctx; + } + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "mapcache module bug? no mapcache_alias_entry found for %s",mapcache_alias); + return NULL; } static mapcache_context_apache_server* apache_server_context_create(server_rec *s, apr_pool_t *pool) @@ -212,6 +244,73 @@ return ctx; } +/* read post body. code taken from "The apache modules book, Nick Kew" */ +static void read_post_body(mapcache_context_apache_request *ctx, mapcache_request_proxy *p) { + request_rec *r = ctx->request; + mapcache_context *mctx = (mapcache_context*)ctx; + int bytes,eos; + apr_bucket_brigade *bb, *bbin; + apr_bucket *b; + apr_status_t rv; + const char *clen = apr_table_get(r->headers_in, "Content-Length"); + if(clen) { + bytes = strtol(clen, NULL, 0); + if(bytes >= p->rule->max_post_len) { + mctx->set_error(mctx, HTTP_REQUEST_ENTITY_TOO_LARGE, "post request too big"); + return; + } + } else { + bytes = p->rule->max_post_len; + } + + bb = apr_brigade_create(mctx->pool, r->connection->bucket_alloc); + bbin = apr_brigade_create(mctx->pool, r->connection->bucket_alloc); + p->post_len = 0; + + do { + apr_bucket *nextb; + rv = ap_get_brigade(r->input_filters, bbin, AP_MODE_READBYTES, APR_BLOCK_READ, bytes); + if(rv != APR_SUCCESS) { + mctx->set_error(mctx, 500, "failed to read form input"); + return; + } + for(b = APR_BRIGADE_FIRST(bbin); b != APR_BRIGADE_SENTINEL(bbin); b = nextb) { + nextb = APR_BUCKET_NEXT(b); + if(APR_BUCKET_IS_EOS(b)) { + eos = 1; + } + if(!APR_BUCKET_IS_METADATA(b)) { + if(b->length != (apr_size_t)(-1)) { + p->post_len += b->length; + if(p->post_len > p->rule->max_post_len) { + apr_bucket_delete(b); + } + } + } + if(p->post_len <= p->rule->max_post_len) { + APR_BUCKET_REMOVE(b); + APR_BRIGADE_INSERT_TAIL(bb, b); + } + } + } while (!eos); + + if(p->post_len > p->rule->max_post_len) { + mctx->set_error(mctx, HTTP_REQUEST_ENTITY_TOO_LARGE, "request too big"); + return; + } + + p->post_buf = apr_palloc(mctx->pool, p->post_len+1); + + if(p->post_len> 0) { + rv = apr_brigade_flatten(bb, p->post_buf, &(p->post_len)); + if(rv != APR_SUCCESS) { + mctx->set_error(mctx, 500, "error (flatten) reading form data"); + return; + } + } + p->post_buf[p->post_len] = 0; +} + static int write_http_response(mapcache_context_apache_request *ctx, mapcache_http_response *response) { request_rec *r = ctx->request; @@ -239,7 +338,7 @@ } } } - if(response->data) { + if(response->data && response->data->size) { ap_set_content_length(r,response->data->size); ap_rwrite((void*)response->data->buf, response->data->size, r); } @@ -251,10 +350,21 @@ static void mod_mapcache_child_init(apr_pool_t *pool, server_rec *s) { - pchild = pool; #ifdef APR_HAS_THREADS apr_thread_mutex_create(&thread_mutex,APR_THREAD_MUTEX_DEFAULT,pool); #endif + for( ; s ; s=s->next) { + mapcache_server_cfg* cfg = ap_get_module_config(s->module_config, &mapcache_module); + int i,rv; + for(i=0;ialiases->nelts;i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + rv = mapcache_connection_pool_create(&(alias_entry->cp),pool); + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, "creating a child process mapcache connection pool on server %s for alias %s", s->server_hostname, alias_entry->endpoint); + if(rv!=APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "failed to create mapcache connection pool"); + } + } + } } static int mod_mapcache_request_handler(request_rec *r) @@ -268,16 +378,25 @@ if (!r->handler || strcmp(r->handler, "mapcache")) { return DECLINED; } - if (r->method_number != M_GET) { + if (r->method_number != M_GET && r->method_number != M_POST) { return HTTP_METHOD_NOT_ALLOWED; } apache_ctx = apache_request_context_create(r); + + if(!apache_ctx) { + return DECLINED; + } + global_ctx = (mapcache_context*)apache_ctx; + global_ctx->supports_redirects = 1; + global_ctx->headers_in = r->headers_in; params = mapcache_http_parse_param_string(global_ctx, r->args); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "mapcache dispatch %s",r->path_info); + mapcache_service_dispatch_request(global_ctx,&request,r->path_info,params,global_ctx->config); if(GC_HAS_ERROR(global_ctx) || !request) { return write_http_response(apache_ctx, @@ -316,7 +435,45 @@ mapcache_request_get_tile *req_tile = (mapcache_request_get_tile*)request; http_response = mapcache_core_get_tile(global_ctx,req_tile); } else if( request->type == MAPCACHE_REQUEST_PROXY ) { + const char *buf; mapcache_request_proxy *req_proxy = (mapcache_request_proxy*)request; + if(r->method_number == M_POST) { + read_post_body(apache_ctx, req_proxy); + if(GC_HAS_ERROR(global_ctx)) { + return write_http_response(apache_ctx, mapcache_core_respond_to_error(global_ctx)); + } + if(!req_proxy->headers) { + req_proxy->headers = apr_table_make(global_ctx->pool, 2); + } + apr_table_set(req_proxy->headers, "Content-Type", r->content_type); + if((buf = apr_table_get(r->headers_in,"X-Forwarded-For"))) { +#if (AP_SERVER_MAJORVERSION_NUMBER == 2) && (AP_SERVER_MINORVERSION_NUMBER < 4) + apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(global_ctx->pool,"%s, %s", buf, r->connection->remote_ip)); +#else + apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(global_ctx->pool,"%s, %s", buf, r->connection->client_ip)); +#endif + } else { +#if (AP_SERVER_MAJORVERSION_NUMBER == 2) && (AP_SERVER_MINORVERSION_NUMBER < 4) + apr_table_set(req_proxy->headers, "X-Forwarded-For", r->connection->remote_ip); +#else + apr_table_set(req_proxy->headers, "X-Forwarded-For", r->connection->client_ip); +#endif + } + if ((buf = apr_table_get(r->headers_in, "Host"))) { + const char *buf2; + if((buf2 = apr_table_get(r->headers_in,"X-Forwarded-Host"))) { + apr_table_set(req_proxy->headers, "X-Forwarded-Host", apr_psprintf(global_ctx->pool,"%s, %s",buf2,buf)); + } else { + apr_table_set(req_proxy->headers, "X-Forwarded-Host", buf); + } + } + + if ((buf = apr_table_get(r->headers_in, "X-Forwarded-Server"))) { + apr_table_set(req_proxy->headers, "X-Forwarded-Server", apr_psprintf(global_ctx->pool, "%s, %s", buf, r->server->server_hostname)); + } else { + apr_table_set(req_proxy->headers, "X-Forwarded-Server", r->server->server_hostname); + } + } http_response = mapcache_core_proxy_request(global_ctx, req_proxy); } else if( request->type == MAPCACHE_REQUEST_GET_MAP) { mapcache_request_get_map *req_map = (mapcache_request_get_map*)request; @@ -389,35 +546,35 @@ return urip - uri; } -static int mapcache_hook_intercept(request_rec *r) +static int mapcache_hook_fixups(request_rec *r) { - mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); - apr_hash_index_t *entry; - - if (!sconfig->aliases) - return DECLINED; - - if (r->uri[0] != '/' && r->uri[0]) - return DECLINED; - - entry = apr_hash_first(r->pool,sconfig->aliases); - - /* loop through the entries to find one where the alias matches */ - while (entry) { - int l = 0; - const char *alias; - apr_ssize_t aliaslen; - mapcache_cfg *c; - apr_hash_this(entry,(const void**)&alias,&aliaslen,(void**)&c); - - if((l=mapcache_alias_matches(r->uri, c->endpoint))>0) { - r->handler = "mapcache"; - r->filename = c->configFile; - r->path_info = &(r->uri[l]); - return OK; + int i; + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "running mapcache fixup on %s (handler:%s,filename:%s)",r->uri,r->handler,r->filename); + if(!r->handler) { + mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias_entry *alias_entry; + + if (!sconfig || !sconfig->aliases) + return DECLINED; + + if (r->uri[0] != '/' && r->uri[0]) + return DECLINED; + + /* loop through the entries to find one where the alias matches */ + for(i=0; ialiases->nelts; i++) { + int l; + alias_entry = APR_ARRAY_IDX(sconfig->aliases,i,mapcache_alias_entry*); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "cheking mapcache alias %s against %s",r->uri,alias_entry->endpoint); + + if((l=mapcache_alias_matches(r->uri, alias_entry->endpoint))>0) { + r->handler = apr_pstrdup(r->pool,"mapcache"); + apr_table_set(r->notes,"mapcache_alias_entry",alias_entry->endpoint); + r->path_info = &(r->uri[l]); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "setting config %s for alias %s",alias_entry->configfile,alias_entry->endpoint); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "using pathinfo %s from uri %s",r->path_info,r->uri); + return OK; + } } - - entry = apr_hash_next(entry); } return DECLINED; @@ -426,21 +583,17 @@ static void mod_mapcache_register_hooks(apr_pool_t *p) { - static const char * const p1[] = { "mod_alias.c", "mod_rewrite.c", NULL }; - static const char * const n1[]= { "mod_userdir.c", - "mod_vhost_alias.c", NULL - }; ap_hook_child_init(mod_mapcache_child_init, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_post_config(mod_mapcache_post_config, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_handler(mod_mapcache_request_handler, NULL, NULL, APR_HOOK_MIDDLE); - ap_hook_translate_name(mapcache_hook_intercept, p1, n1, APR_HOOK_MIDDLE); + ap_hook_fixups(mapcache_hook_fixups, NULL, NULL, APR_HOOK_MIDDLE); } static void* mod_mapcache_create_server_conf(apr_pool_t *pool, server_rec *s) { mapcache_server_cfg *cfg = apr_pcalloc(pool, sizeof(mapcache_server_cfg)); - cfg->aliases = NULL; + cfg->aliases = apr_array_make(pool,1,sizeof(mapcache_alias_entry*)); return cfg; } @@ -451,38 +604,69 @@ mapcache_server_cfg *vhost = (mapcache_server_cfg*)vhost_; mapcache_server_cfg *cfg = apr_pcalloc(p,sizeof(mapcache_server_cfg)); - if (base->aliases && vhost->aliases) { - cfg->aliases = apr_hash_overlay(p, vhost->aliases,base->aliases); - } else if (vhost->aliases) { - cfg->aliases = apr_hash_copy(p,vhost->aliases); - } else if (base->aliases) { - cfg->aliases = apr_hash_copy(p,base->aliases); + cfg->aliases = apr_array_append(p, vhost->aliases,base->aliases); + +#if 0 + { + mapcache_alias_entry *e; + int i; + fprintf(stderr,"#### merge called ####\n"); + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(base->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge base: have alias %s on %s\n",e->configfile,e->endpoint); + } + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(vhost->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge vhosts: have alias %s on %s\n",e->configfile,e->endpoint); + } + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge result: have alias %s on %s\n",e->configfile,e->endpoint); + } } - return vhost; +#endif + return cfg; } static const char* mapcache_add_alias(cmd_parms *cmd, void *cfg, const char *alias, const char* configfile) { - mapcache_server_cfg *sconfig = ap_get_module_config(cmd->server->module_config, &mapcache_module); - mapcache_cfg *config = mapcache_configuration_create(cmd->pool); - mapcache_context *ctx = (mapcache_context*)apache_server_context_create(cmd->server,cmd->pool); - char *msg = NULL; - config->configFile = apr_pstrdup(cmd->pool,configfile); - config->endpoint = alias; - mapcache_configuration_parse(ctx,configfile,config,0); + mapcache_server_cfg *sconfig; + mapcache_alias_entry *alias_entry; + mapcache_context *ctx; + unsigned forbidden = NOT_IN_DIRECTORY|NOT_IN_FILES; + const char *err; + +#if (AP_SERVER_MAJORVERSION_NUMBER > 2) || (AP_SERVER_MINORVERSION_NUMBER >= 4) + forbidden |= NOT_IN_HTACCESS; +#endif + + err = ap_check_cmd_context(cmd, forbidden); + if (err) { + return err; + } + + sconfig = ap_get_module_config(cmd->server->module_config, &mapcache_module); + if(!sconfig || !sconfig->aliases) + return "no mapcache module config, server bug?"; + + alias_entry = apr_pcalloc(cmd->pool,sizeof(mapcache_alias_entry)); + ctx = (mapcache_context*)apache_server_context_create(cmd->server,cmd->pool); + + alias_entry->cfg = mapcache_configuration_create(cmd->pool); + alias_entry->configfile = apr_pstrdup(cmd->pool,configfile); + alias_entry->endpoint = apr_pstrdup(cmd->pool,alias); + mapcache_configuration_parse(ctx,alias_entry->configfile,alias_entry->cfg,0); if(GC_HAS_ERROR(ctx)) { return ctx->get_error_message(ctx); } - mapcache_configuration_post_config(ctx, config); + mapcache_configuration_post_config(ctx, alias_entry->cfg); if(GC_HAS_ERROR(ctx)) { return ctx->get_error_message(ctx); } - ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on alias %s", config->configFile, alias); - if(!sconfig->aliases) { - sconfig->aliases = apr_hash_make(cmd->pool); - } - apr_hash_set(sconfig->aliases,configfile,APR_HASH_KEY_STRING,config); - return msg; + APR_ARRAY_PUSH(sconfig->aliases,mapcache_alias_entry*) = alias_entry; + ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on endpoint %s", alias_entry->configfile, alias_entry->endpoint); + + return NULL; } diff -Nru mapcache-1.2.1/cgi/mapcache.c mapcache-1.4.1/cgi/mapcache.c --- mapcache-1.2.1/cgi/mapcache.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/cgi/mapcache.c 2016-02-25 15:47:16.000000000 +0000 @@ -128,6 +128,11 @@ if(response->mtime) { char *datestr; char *if_modified_since = getenv("HTTP_IF_MODIFIED_SINCE"); + + datestr = apr_palloc(ctx->ctx.pool, APR_RFC822_DATE_LEN); + apr_rfc822_date(datestr, response->mtime); + printf("Last-Modified: %s\r\n", datestr); + if(if_modified_since) { apr_time_t ims_time; apr_int64_t ims,mtime; @@ -138,11 +143,14 @@ ims = apr_time_sec(ims_time); if(ims >= mtime) { printf("Status: 304 Not Modified\r\n"); + /* + * "The 304 response MUST NOT contain a message-body" + * https://tools.ietf.org/html/rfc2616#section-10.3.5 + */ + printf("\r\n"); + return; } } - datestr = apr_palloc(ctx->ctx.pool, APR_RFC822_DATE_LEN); - apr_rfc822_date(datestr, response->mtime); - printf("Last-Modified: %s\r\n", datestr); } if(response->data) { printf("Content-Length: %ld\r\n\r\n", response->data->size); @@ -198,6 +206,7 @@ apr_pool_destroy(config_pool); } config_pool = tmp_config_pool; + mapcache_connection_pool_create(&ctx->connection_pool, config_pool); return; @@ -273,7 +282,6 @@ } } apr_pool_create(&(ctx->pool),config_pool); - ctx->process_pool = config_pool; ctx->threadlock = NULL; request = NULL; pathInfo = getenv("PATH_INFO"); diff -Nru mapcache-1.2.1/cmake/FindAPACHE.cmake mapcache-1.4.1/cmake/FindAPACHE.cmake --- mapcache-1.2.1/cmake/FindAPACHE.cmake 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/cmake/FindAPACHE.cmake 2016-02-25 15:47:16.000000000 +0000 @@ -10,7 +10,7 @@ # find_path(APACHE_INCLUDE_DIR NAMES httpd.h - PATH_SUFFIXES httpd apache apache2 + PATH_SUFFIXES httpd apache apache2 apache22 apache24 ) if(NOT DEFINED APACHE_MODULE_DIR) diff -Nru mapcache-1.2.1/cmake/FindRIAK.cmake mapcache-1.4.1/cmake/FindRIAK.cmake --- mapcache-1.2.1/cmake/FindRIAK.cmake 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/cmake/FindRIAK.cmake 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,14 @@ + +FIND_PATH(RIAK_INCLUDE_DIR + NAMES riack.h +) + +FIND_LIBRARY(RIAK_LIBRARY + NAMES riack +) + +set(RIAK_INCLUDE_DIRS ${RIAK_INCLUDE_DIR}) +set(RIAK_LIBRARIES ${RIAK_LIBRARY}) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(RIAK DEFAULT_MSG RIAK_LIBRARY RIAK_INCLUDE_DIR) +mark_as_advanced(RIAK_LIBRARY RIAK_INCLUDE_DIR) diff -Nru mapcache-1.2.1/CMakeLists.txt mapcache-1.4.1/CMakeLists.txt --- mapcache-1.2.1/CMakeLists.txt 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/CMakeLists.txt 2016-02-25 15:47:16.000000000 +0000 @@ -6,14 +6,13 @@ include(CheckIncludeFile) include(CheckCSourceCompiles) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") add_definitions(-DDEBUG) endif () set (MAPCACHE_VERSION_MAJOR 1) -set (MAPCACHE_VERSION_MINOR 2) +set (MAPCACHE_VERSION_MINOR 4) set (MAPCACHE_VERSION_REVISION 1) @@ -58,9 +57,12 @@ check_function_exists("strncasecmp" HAVE_STRNCASECMP) check_function_exists("symlink" HAVE_SYMLINK) - +check_function_exists ("timegm" HAVE_TIMEGM) set(CMAKE_SKIP_BUILD_RPATH FALSE) +if(APPLE) + set(CMAKE_MACOSX_RPATH ON) +endif() set(CMAKE_LINK_INTERFACE_LIBRARY "") file(GLOB mapcache_SOURCES lib/*.c ) @@ -71,6 +73,15 @@ SOVERSION 1 ) +# Add compiler flags for warnings +if(CMAKE_COMPILER_IS_GNUCC) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=declaration-after-statement") +endif() + +if(CMAKE_C_COMPILER_ID MATCHES "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=declaration-after-statement -std=c89 -Wno-comment") +endif() + #options suported by the cmake builder option(WITH_PIXMAN "Use pixman for SSE optimized image manipulations" ON) option(WITH_SQLITE "Use sqlite as a cache backend" ON) @@ -81,11 +92,12 @@ option(WITH_GEOTIFF "Allow GeoTIFF metadata creation for TIFF cache backends" OFF) option(WITH_PCRE "Use PCRE for regex tests" OFF) option(WITH_MAPSERVER "Enable (experimental) support for the mapserver library" OFF) +option(WITH_RIAK "Use Riak as a cache backend" OFF) find_package(PNG) if(PNG_FOUND) include_directories(${PNG_INCLUDE_DIR}) - target_link_libraries(mapcache ${PNG_LIBRARY}) + target_link_libraries(mapcache ${PNG_LIBRARIES}) else(PNG_FOUND) report_mandatory_not_found(PNG) endif(PNG_FOUND) @@ -156,6 +168,7 @@ include_directories(${PCRE_INCLUDE_DIR}) target_link_libraries(mapcache ${PCRE_LIBRARY}) set (USE_PCRE 1) + add_definitions(-DPCRE_STATIC) else(PCRE_FOUND) report_optional_not_found(PCRE) endif(PCRE_FOUND) @@ -228,6 +241,16 @@ endif(MAPSERVER_FOUND) endif (WITH_MAPSERVER) +if(WITH_RIAK) + find_package(RIAK) + if(RIAK_FOUND) + include_directories(${RIAK_INCLUDE_DIR}) + target_link_libraries(mapcache ${RIAK_LIBRARY}) + set (USE_RIAK 1) + else(RIAK_FOUND) + report_optional_not_found(RIAK) + endif(RIAK_FOUND) +endif (WITH_RIAK) if(UNIX) target_link_libraries(mapcache ${CMAKE_DL_LIBS} m ) @@ -276,6 +299,7 @@ status_optional_component("Experimental TIFF write support" "${USE_TIFF_WRITE}" "${TIFF_LIBRARY}") status_optional_component("PCRE" "${USE_PCRE}" "${PCRE_LIBRARY}") status_optional_component("Experimental mapserver support" "${USE_MAPSERVER}" "${MAPSERVER_LIBRARY}") +status_optional_component("RIAK" "${USE_RIAK}" "${RIAK_LIBRARY}") INSTALL(TARGETS mapcache DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff -Nru mapcache-1.2.1/debian/changelog mapcache-1.4.1/debian/changelog --- mapcache-1.2.1/debian/changelog 2014-06-16 12:54:59.000000000 +0000 +++ mapcache-1.4.1/debian/changelog 2016-11-04 09:30:02.000000000 +0000 @@ -1,8 +1,123 @@ -mapcache (1.2.1-3~trusty1) trusty; urgency=low +mapcache (1.4.1-3~trusty1build1) trusty; urgency=medium - * Launchpad build for Trusty + * No-change rebuild for geos & gdal transitions in ubuntugis-testing. - -- Jerome Villeneuve Larouche Mon, 16 Jun 2014 08:54:28 -0400 + -- Bas Couwenberg Fri, 04 Nov 2016 10:29:42 +0100 + +mapcache (1.4.1-3~trusty1) trusty; urgency=medium + + * Rebuild for trusty. + * Update branch in gbp.conf & Vcs-Git URL. + + -- Bas Couwenberg Thu, 27 Oct 2016 12:55:07 +0200 + +mapcache (1.4.1-3) unstable; urgency=medium + + * Update symbols for hurd-i386. + * Bump Standards-Version to 3.9.8, no changes. + * Drop libmapcache2 post{inst,rm}, rely on dpkg-triggers for ldconfig. + + -- Bas Couwenberg Thu, 05 May 2016 11:40:45 +0200 + +mapcache (1.4.1-2) unstable; urgency=medium + + * Disable memcache support on hurd-i386 too. + * Update symbols for other architectures. + + -- Bas Couwenberg Fri, 26 Feb 2016 09:46:50 +0100 + +mapcache (1.4.1-1) unstable; urgency=medium + + * New upstream release. + * Update Vcs-Git URL to use HTTPS. + * Add patches for various typos. + * Bump Standards-Version to 3.9.7, no changes. + * Update symbols for 1.4.1. + + -- Bas Couwenberg Thu, 25 Feb 2016 21:04:46 +0100 + +mapcache (1.4.0-5~trusty0) trusty; urgency=medium + + * No change rebuild for gdal 2.1.0 transition. + + -- Angelos Tzotsos Mon, 10 May 2016 03:51:00 +0200 + +mapcache (1.4.0-4) unstable; urgency=medium + + * Update watch file to use lowercase RC in uversionmangle. + * Rebuild for geos transition. + + -- Bas Couwenberg Sat, 29 Aug 2015 14:15:23 +0200 + +mapcache (1.4.0-3) unstable; urgency=medium + + * Disable memcache support on architectures where it's not supported. + + -- Bas Couwenberg Tue, 25 Aug 2015 19:22:22 +0200 + +mapcache (1.4.0-2) unstable; urgency=medium + + * Enable memcache support. Thanks to Frederic Junod for the patch. + (closes: 794784) + * Update Vcs-Browser URL to use HTTPS. + * Update Upstream-Contact in copyright file to include email address. + + -- Bas Couwenberg Tue, 25 Aug 2015 16:45:28 +0200 + +mapcache (1.4.0-1) unstable; urgency=medium + + * Update symbols for other architectures. + * Move from experimental to unstable. + + -- Bas Couwenberg Fri, 31 Jul 2015 00:32:07 +0200 + +mapcache (1.4.0-1~exp1) experimental; urgency=medium + + * Update watch file to support new upstream tag format too. + * New upstream release. + * Update copyright file, add license & copyright for hmac-sha.c. + * Refresh patches. + * Disable experimental mapserver support, FTBFS. + * Update symbols for amd64. + * Move docs to libapache2-mod-mapcache. + * Update mapcache_seed man page with new options. + + -- Bas Couwenberg Tue, 28 Jul 2015 21:40:01 +0200 + +mapcache (1.2.1-4) unstable; urgency=medium + + * Update symbols for other architectures. + * Move from experimental to unstable. + + -- Bas Couwenberg Sat, 25 Jul 2015 13:41:00 +0200 + +mapcache (1.2.1-4~exp1) experimental; urgency=medium + + * Add libcurl-ssl-dev as alternative for libcurl-gnutls-dev build dependency. + * Use multi-line uscan options without quotes. + * Update mapserver build dependencies for MapServer 7.0.0. + * Reorder build dependencies, add libharfbuzz-dev. + + -- Bas Couwenberg Fri, 24 Jul 2015 13:45:27 +0200 + +mapcache (1.2.1-3) unstable; urgency=medium + + * Add upstream metadata. + * Update Source URL for GitHub in copyright file. + * Update Homepage URL, strip en/ from path. + * Set the date embedded in man pages to the build date for + reproducible builds. + * Update Vcs-Browser URL to use cgit instead of gitweb. + * Update my email to @debian.org address. + * Bump Standards-Version to 3.9.6, no changes. + * Update copyright file, changes: + - Convert copyright file from iso-8859-1 to utf8 + - Update copyright years + - Drop unused files paragraph for ltmain.sh + * Drop lintian override for no-upstream-changelog, + shouldn't override pedantic tags. + + -- Bas Couwenberg Mon, 27 Apr 2015 12:11:00 +0200 mapcache (1.2.1-2) unstable; urgency=low @@ -38,7 +153,6 @@ * Changed to libgdal-dev b-d. -- Francesco Paolo Lovergine Thu, 12 Dec 2013 17:08:16 +0100 ->>>>>>> debian/master mapcache (1.2.0-1) unstable; urgency=low diff -Nru mapcache-1.2.1/debian/control mapcache-1.4.1/debian/control --- mapcache-1.2.1/debian/control 2014-06-16 12:51:34.000000000 +0000 +++ mapcache-1.4.1/debian/control 2016-10-27 10:54:00.000000000 +0000 @@ -2,44 +2,44 @@ Maintainer: Debian GIS Project Uploaders: Francesco Paolo Lovergine , Alan Boudreault , - Bas Couwenberg + Bas Couwenberg Section: devel Priority: optional Build-Depends: debhelper (>= 9), dpkg-dev (>= 1.16.1.1), - libcurl4-gnutls-dev, - libpng-dev, - libjpeg-dev, - zlib1g-dev, - libgd-dev, + dh-apache2, + apache2-dev, + chrpath, + cmake, + libcairo2-dev, + libcurl4-gnutls-dev | libcurl-ssl-dev, + libfcgi-dev, libfreetype6-dev, + libfribidi-dev, + libgd-dev, libgdal-dev (>= 1.10.0-0~), - libproj-dev, libgeos-dev (>= 3.3.1-1~), - libfribidi-dev, - libcairo2-dev, - librsvg2-dev, - apache2-dev, - dh-apache2, - libfcgi-dev, + libharfbuzz-dev, + libjpeg-dev, + libmemcached-dev, libpcre3-dev, libpixman-1-dev, + libpng-dev, + libproj-dev, + librsvg2-dev, libsqlite3-dev, libtiff-dev, - libmapserver1 (>= 6.4.0-2), - libmapserver1-dev (>= 6.4.0-2), - chrpath, - cmake, + zlib1g-dev, pkg-config, pkg-kde-tools, docbook2x, docbook-xsl, docbook-xml, xsltproc -Standards-Version: 3.9.5 -Vcs-Browser: http://anonscm.debian.org/gitweb/?p=pkg-grass/mapcache.git -Vcs-Git: git://anonscm.debian.org/pkg-grass/mapcache.git -Homepage: http://www.mapserver.org/en/mapcache/ +Standards-Version: 3.9.8 +Vcs-Browser: https://anonscm.debian.org/cgit/pkg-grass/mapcache.git +Vcs-Git: https://anonscm.debian.org/git/pkg-grass/mapcache.git -b ubuntu/trusty +Homepage: http://www.mapserver.org/mapcache/ Package: libmapcache1 Architecture: any diff -Nru mapcache-1.2.1/debian/copyright mapcache-1.4.1/debian/copyright --- mapcache-1.2.1/debian/copyright 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/copyright 2016-03-07 19:51:35.000000000 +0000 @@ -1,33 +1,30 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: MapCache -Upstream-Contact: Thomas Bonfort and the MapServer team. -Source: http://mapserver.org/ -License: MIT/X11 -Copyright: 1996-2012 Regents of the University of Minnesota. +Upstream-Contact: The MapServer team +Source: https://github.com/mapserver/mapcache/releases Files: * -Copyright: 1996-2011, Regents of the University of Minnesota - 2011, Regents of the University of Minnesota -License: MIT/X11 +Copyright: 1996-2013, Regents of the University of Minnesota +License: MIT Files: include/ezxml.h lib/ezxml.c Copyright: 2004-2006, Aaron Voisine License: Expat +Files: lib/hmac-sha.c +Copyright: 2005, 2007, Olivier Gay +License: BSD-3-clause + Files: lib/strptime.c Copyright: 1999, Kungliga Tekniska Högskolan License: BSD-3-clause -Files: ltmain.sh -Copyright: 1996-2001, 2003-2006 -License: GPL-2+ - Files: debian/* Copyright: 2012, Tom Payne License: BSD-3-clause -License: MIT/X11 +License: MIT Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -93,24 +90,3 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -License: GPL-2+ - This program is free software; you can redistribute it - and/or modify it under the terms of the GNU General Public - License as published by the Free Software Foundation; either - version 2 of the License, or (at your option) any later - version. - . - This program is distributed in the hope that it will be - useful, but WITHOUT ANY WARRANTY; without even the implied - warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - PURPOSE. See the GNU General Public License for more - details. - . - You should have received a copy of the GNU General Public - License along with this package; if not, write to the Free - Software Foundation, Inc., 51 Franklin St, Fifth Floor, - Boston, MA 02110-1301 USA - . - On Debian systems, the full text of the GNU General Public - License version 2 can be found in the file - `/usr/share/common-licenses/GPL-2'. diff -Nru mapcache-1.2.1/debian/docs mapcache-1.4.1/debian/docs --- mapcache-1.2.1/debian/docs 2013-06-17 18:36:19.000000000 +0000 +++ mapcache-1.4.1/debian/docs 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -README diff -Nru mapcache-1.2.1/debian/gbp.conf mapcache-1.4.1/debian/gbp.conf --- mapcache-1.2.1/debian/gbp.conf 2014-06-16 12:51:44.000000000 +0000 +++ mapcache-1.4.1/debian/gbp.conf 2016-10-27 10:53:48.000000000 +0000 @@ -6,7 +6,7 @@ # The default name for the Debian branch is "master". # Change it if the name is different (for instance, "debian/unstable"). -debian-branch = master +debian-branch = ubuntu/trusty # git-import-orig uses the following names for the upstream tags. # Change the value if you are not using git-import-orig diff -Nru mapcache-1.2.1/debian/libapache2-mod-mapcache.docs mapcache-1.4.1/debian/libapache2-mod-mapcache.docs --- mapcache-1.2.1/debian/libapache2-mod-mapcache.docs 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/debian/libapache2-mod-mapcache.docs 2016-02-07 00:15:11.000000000 +0000 @@ -0,0 +1 @@ +README diff -Nru mapcache-1.2.1/debian/libapache2-mod-mapcache.lintian-overrides mapcache-1.4.1/debian/libapache2-mod-mapcache.lintian-overrides --- mapcache-1.2.1/debian/libapache2-mod-mapcache.lintian-overrides 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/libapache2-mod-mapcache.lintian-overrides 2016-03-07 19:51:35.000000000 +0000 @@ -1,3 +1,3 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -libapache2-mod-mapcache: no-upstream-changelog +# False positive, dependency set by dh_apache2 +libapache2-mod-mapcache: apache2-module-depends-on-real-apache2-package apache2-bin + diff -Nru mapcache-1.2.1/debian/libmapcache1-dev.lintian-overrides mapcache-1.4.1/debian/libmapcache1-dev.lintian-overrides --- mapcache-1.2.1/debian/libmapcache1-dev.lintian-overrides 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/libmapcache1-dev.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -libmapcache1-dev: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/libmapcache1.lintian-overrides mapcache-1.4.1/debian/libmapcache1.lintian-overrides --- mapcache-1.2.1/debian/libmapcache1.lintian-overrides 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/libmapcache1.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -libmapcache1: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/libmapcache1.postinst mapcache-1.4.1/debian/libmapcache1.postinst --- mapcache-1.2.1/debian/libmapcache1.postinst 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/libmapcache1.postinst 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -#DEBHELPER# - -if [ "$1" = configure ] ; then - ldconfig -fi - -exit 0 diff -Nru mapcache-1.2.1/debian/libmapcache1.postrm mapcache-1.4.1/debian/libmapcache1.postrm --- mapcache-1.2.1/debian/libmapcache1.postrm 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/libmapcache1.postrm 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -#DEBHELPER# - -if [ "$1" = remove ] ; then - ldconfig -fi - -exit 0 diff -Nru mapcache-1.2.1/debian/libmapcache1.symbols mapcache-1.4.1/debian/libmapcache1.symbols --- mapcache-1.2.1/debian/libmapcache1.symbols 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/libmapcache1.symbols 2016-10-27 10:52:52.000000000 +0000 @@ -1,4 +1,4 @@ -# SymbolsHelper-Confirmed: 1.2.0 amd64 +# SymbolsHelper-Confirmed: 1.4.1 alpha arm64 armel armhf hppa hurd-i386 i386 kfreebsd-amd64 kfreebsd-i386 mips mips64el mipsel ppc64el x32 libmapcache.so.1 #PACKAGE# #MINVER# EZXML_NIL@Base 1.0.0 _configuration_parse_wms_xml@Base 1.0.0 @@ -20,9 +20,12 @@ _create_demo_wmts@Base 1.0.0 _error_report_wmts@Base 1.0.0 _format_error_wms@Base 1.0.0 + _mapcache_cache_rest_headers@Base 1.4.0 _mapcache_context_clear_error_default@Base 1.0.0 _mapcache_context_get_error_default@Base 1.0.0 _mapcache_context_get_error_msg_default@Base 1.0.0 + _mapcache_context_pop_errors@Base 1.4.0 + _mapcache_context_push_errors@Base 1.4.0 _mapcache_context_set_error_default@Base 1.0.0 _mapcache_context_set_exception_default@Base 1.0.0 _mapcache_curl_header_callback@Base 1.0.0 @@ -56,10 +59,6 @@ _mapcache_source_dummy_configuration_parse_xml@Base 1.0.0 _mapcache_source_dummy_query@Base 1.0.0 _mapcache_source_dummy_render_map@Base 1.0.0 - _mapcache_source_mapserver_configuration_check@Base 1.0.0 - _mapcache_source_mapserver_configuration_parse_xml@Base 1.0.0 - _mapcache_source_mapserver_query@Base 1.0.0 - _mapcache_source_mapserver_render_map@Base 1.0.0 _mapcache_source_wms_configuration_check@Base 1.0.0 _mapcache_source_wms_configuration_parse_xml@Base 1.0.0 _mapcache_source_wms_query@Base 1.0.0 @@ -70,7 +69,12 @@ _mapcache_unescape_url@Base 1.0.0 _wmts_inspire_metadata_responselanguages@Base 1.2.0 _wmts_service_identification_keywords@Base 1.2.0 + base64_encode@Base 1.4.0 + buffer_write_callback@Base 1.4.0 crc_table@Base 1.2.0 + (arch=!alpha !amd64 !arm64 !armel !armhf !hurd-i386 !i386 !kfreebsd-amd64 !kfreebsd-i386 !mips64el !mipsel !ppc64el !x32)crc_table_computed@Base 1.4.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)create_memcache@Base 1.4.1 + empty_png_384@Base 1.4.0 ezxml_add_child@Base 1.0.0 ezxml_ampencode@Base 1.0.0 ezxml_attr@Base 1.0.0 @@ -104,14 +108,30 @@ ezxml_toxml_r@Base 1.0.0 ezxml_vget@Base 1.0.0 fontData@Base 1.0.0 + hmac_sha1@Base 1.4.0 + hmac_sha256@Base 1.4.0 + hmac_sha256_final@Base 1.4.0 + hmac_sha256_init@Base 1.4.0 + hmac_sha256_reinit@Base 1.4.0 + hmac_sha256_update@Base 1.4.0 lock_filename_for_resource@Base 1.0.0 mapcache_assemble_maps@Base 1.2.0 mapcache_buffer_append@Base 1.0.0 mapcache_buffer_create@Base 1.0.0 + mapcache_cache_azure_create@Base 1.4.0 + mapcache_cache_composite_create@Base 1.4.0 mapcache_cache_disk_create@Base 1.0.0 + mapcache_cache_fallback_create@Base 1.4.0 + mapcache_cache_google_create@Base 1.4.0 mapcache_cache_mbtiles_create@Base 1.0.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_cache_memcache_create@Base 1.4.1 + mapcache_cache_multitier_create@Base 1.4.0 + mapcache_cache_rest_create@Base 1.4.0 + mapcache_cache_rest_init@Base 1.4.0 + mapcache_cache_s3_create@Base 1.4.0 mapcache_cache_sqlite_create@Base 1.0.0 mapcache_cache_tiff_create@Base 1.0.0 + mapcache_config_parse_locker@Base 1.4.0 mapcache_configuration_add_cache@Base 1.0.0 mapcache_configuration_add_grid@Base 1.0.0 mapcache_configuration_add_image_format@Base 1.0.0 @@ -126,6 +146,10 @@ mapcache_configuration_parse@Base 1.0.0 mapcache_configuration_parse_xml@Base 1.0.0 mapcache_configuration_post_config@Base 1.0.0 + mapcache_connection_pool_create@Base 1.4.0 + mapcache_connection_pool_get_connection@Base 1.4.0 + mapcache_connection_pool_invalidate_connection@Base 1.4.0 + mapcache_connection_pool_release_connection@Base 1.4.0 mapcache_context_copy@Base 1.0.0 mapcache_context_init@Base 1.0.0 mapcache_core_get_capabilities@Base 1.0.0 @@ -136,15 +160,15 @@ mapcache_core_respond_to_error@Base 1.0.0 mapcache_dimension_intervals_create@Base 1.0.0 mapcache_dimension_regex_create@Base 1.0.0 + mapcache_dimension_sqlite_create@Base 1.4.0 mapcache_dimension_time_create@Base 1.0.0 mapcache_dimension_values_create@Base 1.0.0 mapcache_empty_png_decode@Base 1.2.0 mapcache_error_image@Base 1.0.0 -#MISSING: 1.2.0# mapcache_fetch_maps@Base 1.0.0 mapcache_grid_compute_limits@Base 1.0.0 mapcache_grid_create@Base 1.0.0 mapcache_grid_get_cell@Base 1.0.0 - mapcache_grid_get_closest_level@Base 1.0.0 + mapcache_grid_get_closest_wms_level@Base 1.4.0 mapcache_grid_get_crs@Base 1.0.0 mapcache_grid_get_extent@Base 1.0.0 mapcache_grid_get_horizontal_resolution@Base 1.0.0 @@ -166,6 +190,7 @@ mapcache_image_create@Base 1.0.0 mapcache_image_create_empty@Base 1.0.0 mapcache_image_create_with_data@Base 1.2.0 + mapcache_image_fill@Base 1.4.0 mapcache_image_has_alpha@Base 1.0.0 mapcache_image_merge@Base 1.0.0 mapcache_image_metatile_split@Base 1.0.0 @@ -179,6 +204,24 @@ mapcache_imageio_is_valid_format@Base 1.0.0 mapcache_is_axis_inverted@Base 1.0.0 mapcache_lock_or_wait_for_resource@Base 1.0.0 + mapcache_locker_disk_aquire_lock@Base 1.4.0 + mapcache_locker_disk_create@Base 1.4.0 + mapcache_locker_disk_parse_xml@Base 1.4.0 + mapcache_locker_disk_ping_lock@Base 1.4.0 + mapcache_locker_disk_release_lock@Base 1.4.0 + mapcache_locker_fallback_aquire_lock@Base 1.4.0 + mapcache_locker_fallback_create@Base 1.4.0 + mapcache_locker_fallback_parse_xml@Base 1.4.0 + mapcache_locker_fallback_ping_lock@Base 1.4.0 + mapcache_locker_fallback_release_lock@Base 1.4.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_aquire_lock@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_create@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_parse_xml@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_ping_lock@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_release_lock@Base 1.4.1 + mapcache_make_parent_dirs@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_memcache_connection_constructor@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_memcache_connection_destructor@Base 1.4.1 mapcache_meters_per_unit@Base 1.0.0 mapcache_ogc_strptime@Base 1.2.0 mapcache_prefetch_tiles@Base 1.0.0 @@ -196,6 +239,10 @@ mapcache_source_init@Base 1.0.0 mapcache_source_mapserver_create@Base 1.0.0 mapcache_source_wms_create@Base 1.0.0 + mapcache_sqlite_connection_constructor@Base 1.4.0 + mapcache_sqlite_connection_destructor@Base 1.4.0 + mapcache_sqlite_dimension_connection_constructor@Base 1.4.0 + mapcache_sqlite_dimension_connection_destructor@Base 1.4.0 mapcache_tileset_add_watermark@Base 1.0.0 mapcache_tileset_assemble_map_tiles@Base 1.0.0 mapcache_tileset_assemble_out_of_zoom_tile@Base 1.2.0 @@ -234,3 +281,14 @@ parseTileset@Base 1.0.0 parseTimeDimension@Base 1.2.0 relative_path@Base 1.0.0 + sha256@Base 1.4.0 + sha256_final@Base 1.4.0 + sha256_h0@Base 1.4.0 + sha256_init@Base 1.4.0 + sha256_k@Base 1.4.0 + sha256_transf@Base 1.4.0 + sha256_update@Base 1.4.0 + sha_hex_encode@Base 1.4.0 + timegm@Base 1.4.1 + to_hex@Base 1.4.0 + url_encode@Base 1.4.0 diff -Nru mapcache-1.2.1/debian/mapcache-cgi.lintian-overrides mapcache-1.4.1/debian/mapcache-cgi.lintian-overrides --- mapcache-1.2.1/debian/mapcache-cgi.lintian-overrides 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/mapcache-cgi.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -mapcache-cgi: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/mapcache_seed.1.xml mapcache-1.4.1/debian/mapcache_seed.1.xml --- mapcache-1.2.1/debian/mapcache_seed.1.xml 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/mapcache_seed.1.xml 2016-10-27 10:50:38.000000000 +0000 @@ -45,6 +45,14 @@ + + override + + Override cache used by selected tileset (useful for selectively seeding fallback/multitier caches). + + + + name @@ -124,6 +132,30 @@ + + + number + + Percent of failed requests allowed from the last 1000 before we abort (default: 1%, set to 0 to abort on first error). + + + + + + file + + Log failed tiles to file. + + + + + + file + + Rtry failed requests logged to file by . + + + seed|delete|transfer diff -Nru mapcache-1.2.1/debian/mapcache-tools.lintian-overrides mapcache-1.4.1/debian/mapcache-tools.lintian-overrides --- mapcache-1.2.1/debian/mapcache-tools.lintian-overrides 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/mapcache-tools.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -mapcache-tools: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/patches/across-typo.patch mapcache-1.4.1/debian/patches/across-typo.patch --- mapcache-1.2.1/debian/patches/across-typo.patch 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/debian/patches/across-typo.patch 2016-10-27 10:52:52.000000000 +0000 @@ -0,0 +1,16 @@ +Description: Fix 'accross' typo, replace with 'across'. +Author: Bas Couwenberg +Forwarded: https://github.com/mapserver/mapcache/pull/141 +Applied-Upstream: https://github.com/mapserver/mapcache/commit/ac20472ef5385bf205c500a4f6793453ecaddac5 + +--- a/lib/service_demo.c ++++ b/lib/service_demo.c +@@ -282,7 +282,7 @@ static char *demo_head_gmaps = + " return null;\n" + " }\n" + "\n" +- " // repeat accross x-axis\n" ++ " // repeat across x-axis\n" + " if (x < 0 || x >= tileRange) {\n" + " x = (x % tileRange + tileRange) % tileRange;\n" + " }\n" diff -Nru mapcache-1.2.1/debian/patches/cmake-mapserver-include.patch mapcache-1.4.1/debian/patches/cmake-mapserver-include.patch --- mapcache-1.2.1/debian/patches/cmake-mapserver-include.patch 2014-06-16 12:51:34.000000000 +0000 +++ mapcache-1.4.1/debian/patches/cmake-mapserver-include.patch 2016-10-27 10:52:52.000000000 +0000 @@ -1,9 +1,10 @@ Description: Include mapserver cmake package for additional include directories. -Author: Bas Couwenberg -Last-Update: 2013-10-21 +Author: Bas Couwenberg +Forwarded: not-needed + --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -218,10 +218,18 @@ if(WITH_GEOTIFF) +@@ -231,10 +231,18 @@ if(WITH_GEOTIFF) endif (WITH_GEOTIFF) if(WITH_MAPSERVER) diff -Nru mapcache-1.2.1/debian/patches/occurred-typo.patch mapcache-1.4.1/debian/patches/occurred-typo.patch --- mapcache-1.2.1/debian/patches/occurred-typo.patch 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/debian/patches/occurred-typo.patch 2016-10-27 10:52:52.000000000 +0000 @@ -0,0 +1,38 @@ +Description: Fix 'occured' typo, replace with 'occurred'. +Author: Bas Couwenberg +Forwarded: https://github.com/mapserver/mapcache/pull/141 +Applied-Upstream: https://github.com/mapserver/mapcache/commit/ac20472ef5385bf205c500a4f6793453ecaddac5 + +--- a/include/mapcache.h ++++ b/include/mapcache.h +@@ -207,7 +207,7 @@ struct mapcache_context { + void (*set_exception)(mapcache_context *ctx, char *key, char *message, ...); + + /** +- * \brief query context to know if an error has occured ++ * \brief query context to know if an error has occurred + * \memberof mapcache_context + */ + int (*get_error)(mapcache_context * ctx); +--- a/lib/cache_tiff.c ++++ b/lib/cache_tiff.c +@@ -317,7 +317,7 @@ static int _mapcache_cache_tiff_get(mapc + * we currrently have no way of knowing if the opening failed because the tif + * file does not exist (which is not an error condition, as it only signals + * that the requested tile does not exist in the cache), or if an other error +- * that should be signaled occured (access denied, not a tiff file, etc...) ++ * that should be signaled occurred (access denied, not a tiff file, etc...) + * + * we ignore this case here and hope that further parts of the code will be + * able to detect what's happening more precisely +--- a/lib/core.c ++++ b/lib/core.c +@@ -620,7 +620,7 @@ mapcache_http_response* mapcache_core_re + + msg = ctx->_errmsg; + if(!msg) { +- msg = apr_pstrdup(ctx->pool,"an unspecified error has occured"); ++ msg = apr_pstrdup(ctx->pool,"an unspecified error has occurred"); + } + ctx->log(ctx,MAPCACHE_ERROR,msg); + diff -Nru mapcache-1.2.1/debian/patches/overridden-typo.patch mapcache-1.4.1/debian/patches/overridden-typo.patch --- mapcache-1.2.1/debian/patches/overridden-typo.patch 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/debian/patches/overridden-typo.patch 2016-10-27 10:52:52.000000000 +0000 @@ -0,0 +1,16 @@ +Description: Fix 'overrided' typo, replace with 'overridden'. +Author: Bas Couwenberg +Forwarded: https://github.com/mapserver/mapcache/pull/141 +Applied-Upstream: https://github.com/mapserver/mapcache/commit/ac20472ef5385bf205c500a4f6793453ecaddac5 + +--- a/util/mapcache_seed.c ++++ b/util/mapcache_seed.c +@@ -1153,7 +1153,7 @@ int main(int argc, const char **argv) + if(cache_override) { + mapcache_cache *co = mapcache_configuration_get_cache(cfg, cache_override); + if(!co) { +- return usage(argv[0], "overrided cache\"%s\" to not found in configuration", cache_override); ++ return usage(argv[0], "overridden cache\"%s\" not found in configuration", cache_override); + } else { + tileset->_cache = co; + } diff -Nru mapcache-1.2.1/debian/patches/series mapcache-1.4.1/debian/patches/series --- mapcache-1.2.1/debian/patches/series 2014-06-16 12:51:34.000000000 +0000 +++ mapcache-1.4.1/debian/patches/series 2016-10-27 10:52:52.000000000 +0000 @@ -1 +1,4 @@ cmake-mapserver-include.patch +across-typo.patch +occurred-typo.patch +overridden-typo.patch diff -Nru mapcache-1.2.1/debian/rules mapcache-1.4.1/debian/rules --- mapcache-1.2.1/debian/rules 2014-06-16 12:51:44.000000000 +0000 +++ mapcache-1.4.1/debian/rules 2016-10-27 10:52:52.000000000 +0000 @@ -8,9 +8,10 @@ export DH_OPTIONS # Enable hardening build flags -#export DEB_BUILD_MAINT_OPTIONS=hardening=+all +export DEB_BUILD_MAINT_OPTIONS=hardening=+all -DEB_HOST_MULTIARCH?=$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) +DEB_HOST_ARCH ?= $(shell dpkg-architecture -qDEB_HOST_ARCH) +DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) CPPFLAGS:=$(shell dpkg-buildflags --get CPPFLAGS) CFLAGS:=$(shell dpkg-buildflags --get CFLAGS) @@ -19,7 +20,8 @@ CFLAGS+=$(CPPFLAGS) CFLAGS+=$(LDFLAGS) -MAPCACHE_VERSION = $(shell dpkg-parsechangelog | sed -ne 's/^Version: \(.*\)-.*/\1/p' | sed -e 's/^\(.*\)~.*/\1/' | sed -e 's/dev.*/.0/') +MAPCACHE_VERSION=$(shell dpkg-parsechangelog | sed -ne 's/^Version: \(.*\)-.*/\1/p' | sed -e 's/\+.*//; s/^[0-9]://') +BUILD_DATE=$(shell dpkg-parsechangelog | sed -ne 's/^Date: //p' | LC_ALL=C date -u "+%d %B %Y" -f -) CMAKE_OPTS:= \ -DCMAKE_INSTALL_PREFIX=/usr \ @@ -27,11 +29,10 @@ -DWITH_PIXMAN=1 \ -DWITH_SQLITE=1 \ -DWITH_BERKELEY_DB=0 \ - -DWITH_MEMCACHE=0 \ -DWITH_TIFF=1 \ -DWITH_TIFF_WRITE_SUPPORT=0 \ -DWITH_GEOTIFF=0 \ - -DWITH_MAPSERVER=1 \ + -DWITH_MAPSERVER=0 \ -DWITH_PCRE=1 \ -DWITH_APACHE=1 \ -DWITH_VERSION_STRING=1 \ @@ -40,6 +41,13 @@ -DWITH_GEOS=1 \ -DWITH_OGR=1 +# Disable memcache support on architectures where it's not supported +ifneq (,$(findstring $(DEB_HOST_ARCH),"armel armhf i386 mips mipsel powerpc hppa hurd-i386")) + CMAKE_OPTS += -DWITH_MEMCACHE=0 +else + CMAKE_OPTS += -DWITH_MEMCACHE=1 +endif + %: dh $@ --with apache2,pkgkde_symbolshelper \ --parallel \ @@ -57,7 +65,7 @@ override_dh_auto_build: # Create man page from DocBook XML - -docbook2x-man $(CURDIR)/debian/mapcache_seed.1.xml + -docbook2x-man --string-param header-3="$(BUILD_DATE)" $(CURDIR)/debian/mapcache_seed.1.xml -mv mapcache_seed.1 $(CURDIR)/debian/ dh_auto_build diff -Nru mapcache-1.2.1/debian/upstream/metadata mapcache-1.4.1/debian/upstream/metadata --- mapcache-1.2.1/debian/upstream/metadata 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/debian/upstream/metadata 2016-02-07 00:15:11.000000000 +0000 @@ -0,0 +1,7 @@ +--- +Bug-Database: https://github.com/mapserver/mapcache/issues +Bug-Submit: https://github.com/mapserver/mapcache/issues/new +Contact: The MapServer team +Name: MapCache +Repository: https://github.com/mapserver/mapcache.git +Repository-Browse: https://github.com/mapserver/mapcache diff -Nru mapcache-1.2.1/debian/watch mapcache-1.4.1/debian/watch --- mapcache-1.2.1/debian/watch 2014-01-16 19:46:10.000000000 +0000 +++ mapcache-1.4.1/debian/watch 2016-03-07 19:51:35.000000000 +0000 @@ -1,3 +1,7 @@ version=3 -opts="dversionmangle=s/\+(debian|dfsg|ds|deb)\d*$//,uversionmangle=s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha)\d*)$/$1~$2/; s/\-/\./g" \ -https://github.com/mapserver/mapcache/releases (?:.*/)?(?:rel|v|mapcache)[\-\_](\d[\d\-\.]+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) +opts=\ +dversionmangle=s/\+(debian|dfsg|ds|deb)\d*$//,\ +uversionmangle=s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha)\d*)$/$1~$2/;s/RC/rc/;s/\-/\./g,\ +filenamemangle=s/(?:.*?)?(?:rel|v|mapcache)?[\-\_]?(\d\S+)\.(tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))/mapcache-$1.$2/ \ +https://github.com/mapserver/mapcache/releases \ +(?:.*?/)?(?:rel|v|mapcache)?[\-\_]?(\d[\d\-\.]+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) diff -Nru mapcache-1.2.1/.gitignore mapcache-1.4.1/.gitignore --- mapcache-1.2.1/.gitignore 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/.gitignore 2016-02-25 15:47:16.000000000 +0000 @@ -1,3 +1,5 @@ .*.swp nbproject/ /build/ +/build_vagrant/ +/.vagrant/ diff -Nru mapcache-1.2.1/include/mapcache-config.h.in mapcache-1.4.1/include/mapcache-config.h.in --- mapcache-1.2.1/include/mapcache-config.h.in 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/include/mapcache-config.h.in 2016-02-25 15:47:16.000000000 +0000 @@ -11,6 +11,7 @@ #cmakedefine USE_GEOTIFF 1 #cmakedefine USE_PCRE 1 #cmakedefine USE_MAPSERVER 1 +#cmakedefine USE_RIAK 1 #cmakedefine HAVE_STRNCASECMP 1 #cmakedefine HAVE_SYMLINK 1 diff -Nru mapcache-1.2.1/include/mapcache.h mapcache-1.4.1/include/mapcache.h --- mapcache-1.2.1/include/mapcache.h 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/include/mapcache.h 2016-02-25 15:47:16.000000000 +0000 @@ -39,6 +39,7 @@ #include #include +#include #include "util.h" #include "ezxml.h" @@ -66,6 +67,14 @@ #include #endif +#ifdef USE_COUCHBASE +#include +#endif + +#ifdef USE_RIAK +#include +#endif + #define MAPCACHE_SUCCESS 0 #define MAPCACHE_FAILURE 1 #define MAPCACHE_TRUE 1 @@ -75,11 +84,17 @@ #define MAPCACHE_TILESET_WRONG_EXTENT 4 #define MAPCACHE_CACHE_MISS 5 #define MAPCACHE_FILE_LOCKED 6 +#define MAPCACHE_CACHE_RELOAD 7 -#define MAPCACHE_USERAGENT "mod-mapcache/"MAPCACHE_VERSION +#define MAPCACHE_MAX_NUM_TILES 1000 -#define MAPCACHE_LOCKFILE_PREFIX "_gc_lock" +#define MAPCACHE_USERAGENT "mod-mapcache/"MAPCACHE_VERSION +#if defined(_WIN32) && !defined(__CYGWIN__) +# define MS_DLL_EXPORT __declspec(dllexport) +#else +#define MS_DLL_EXPORT +#endif typedef struct mapcache_image_format mapcache_image_format; @@ -103,11 +118,19 @@ typedef struct mapcache_source_gdal mapcache_source_gdal; #endif typedef struct mapcache_cache_disk mapcache_cache_disk; +typedef struct mapcache_cache_composite mapcache_cache_composite; +typedef struct mapcache_cache_fallback mapcache_cache_fallback; +typedef struct mapcache_cache_multitier mapcache_cache_multitier; +typedef struct mapcache_cache_rest mapcache_cache_rest; +typedef struct mapcache_cache_s3 mapcache_cache_s3; +typedef struct mapcache_cache_azure mapcache_cache_azure; +typedef struct mapcache_cache_google mapcache_cache_google; #ifdef USE_TIFF typedef struct mapcache_cache_tiff mapcache_cache_tiff; #endif typedef struct mapcache_http mapcache_http; typedef struct mapcache_request mapcache_request; +typedef struct mapcache_request_image mapcache_request_image; typedef struct mapcache_request_proxy mapcache_request_proxy; typedef struct mapcache_request_get_capabilities mapcache_request_get_capabilities; typedef struct mapcache_request_get_capabilities_demo mapcache_request_get_capabilities_demo; @@ -139,9 +162,12 @@ typedef struct mapcache_timedimension mapcache_timedimension; typedef struct mapcache_dimension_intervals mapcache_dimension_intervals; typedef struct mapcache_dimension_values mapcache_dimension_values; +typedef struct mapcache_dimension_sqlite mapcache_dimension_sqlite; typedef struct mapcache_dimension_regex mapcache_dimension_regex; typedef struct mapcache_extent mapcache_extent; typedef struct mapcache_extent_i mapcache_extent_i; +typedef struct mapcache_connection_pool mapcache_connection_pool; +typedef struct mapcache_locker mapcache_locker; /** \defgroup utility Utility */ /** @{ */ @@ -198,6 +224,18 @@ */ void (*clear_errors)(mapcache_context * ctx); + /** + * \brief clear current error and store it in mapcache_error + * \memberof mapcache_context + */ + void (*pop_errors)(mapcache_context * ctx, void **error); + + /** + * \brief restore error status from mapcache_error + * \memberof mapcache_context + */ + void (*push_errors)(mapcache_context * ctx, void *error); + /** * \brief log a message @@ -208,7 +246,7 @@ const char* (*get_instance_id)(mapcache_context * ctx); mapcache_context* (*clone)(mapcache_context *ctx); apr_pool_t *pool; - apr_pool_t *process_pool; + mapcache_connection_pool *connection_pool; void *threadlock; char *_contenttype; char *_errmsg; @@ -216,12 +254,14 @@ mapcache_cfg *config; mapcache_service *service; apr_table_t *exceptions; + int supports_redirects; + apr_table_t *headers_in; }; -void mapcache_context_init(mapcache_context *ctx); -void mapcache_context_copy(mapcache_context *src, mapcache_context *dst); +MS_DLL_EXPORT void mapcache_context_init(mapcache_context *ctx); +MS_DLL_EXPORT void mapcache_context_copy(mapcache_context *src, mapcache_context *dst); -#define GC_CHECK_ERROR_RETURN(ctx) (if(((mapcache_context*)ctx)->_errcode) return MAPCACHE_FAILURE;) +#define GC_CHECK_ERROR_RETURN(ctx) if(((mapcache_context*)ctx)->_errcode) return MAPCACHE_FAILURE; #define GC_CHECK_ERROR(ctx) if(((mapcache_context*)ctx)->_errcode) return; #define GC_HAS_ERROR(ctx) (((mapcache_context*)ctx)->_errcode > 0) @@ -299,6 +339,8 @@ struct mapcache_http { char *url; /**< the base url to request */ apr_table_t *headers; /**< additional headers to add to the http request, eg, Referer */ + char *post_body; + size_t post_len; int connection_timeout; int timeout; /* TODO: authentication */ @@ -355,7 +397,8 @@ /** @{ */ typedef enum { - MAPCACHE_CACHE_DISK + MAPCACHE_CACHE_DISK, + MAPCACHE_CACHE_REST #ifdef USE_MEMCACHE ,MAPCACHE_CACHE_MEMCACHE #endif @@ -371,6 +414,13 @@ #ifdef USE_TIFF ,MAPCACHE_CACHE_TIFF #endif + ,MAPCACHE_CACHE_COMPOSITE +#ifdef USE_COUCHBASE + ,MAPCACHE_CACHE_COUCHBASE +#endif +#ifdef USE_RIAK + ,MAPCACHE_CACHE_RIAK +#endif } mapcache_cache_type; /** \interface mapcache_cache @@ -388,23 +438,23 @@ * \returns MAPCACHE_CACHE_MISS if the file does not exist on the disk * \memberof mapcache_cache */ - int (*tile_get)(mapcache_context *ctx, mapcache_tile * tile); + int (*tile_get)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); /** * delete tile from cache * * \memberof mapcache_cache */ - void (*tile_delete)(mapcache_context *ctx, mapcache_tile * tile); + void (*tile_delete)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); - int (*tile_exists)(mapcache_context *ctx, mapcache_tile * tile); + int (*tile_exists)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); /** * set tile content to cache * \memberof mapcache_cache */ - void (*tile_set)(mapcache_context *ctx, mapcache_tile * tile); - void (*tile_multi_set)(mapcache_context *ctx, mapcache_tile *tiles, int ntiles); + void (*tile_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); + void (*tile_multi_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles); void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_cache * cache, mapcache_cfg *config); void (*configuration_post_config)(mapcache_context *ctx, mapcache_cache * cache, mapcache_cfg *config); @@ -425,7 +475,109 @@ * Set filename for a given tile * \memberof mapcache_cache_disk */ - void (*tile_key)(mapcache_context *ctx, mapcache_tile *tile, char **path); + void (*tile_key)(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path); +}; + +typedef struct mapcache_cache_composite_cache_link mapcache_cache_composite_cache_link; +struct mapcache_cache_composite_cache_link { + mapcache_cache *cache; + int minzoom; + int maxzoom; + apr_array_header_t *grids; + apr_array_header_t *dimensions; //TODO +}; + +struct mapcache_cache_composite { + mapcache_cache cache; + apr_array_header_t *cache_links; +}; + +struct mapcache_cache_fallback { + mapcache_cache cache; + apr_array_header_t *caches; +}; + +struct mapcache_cache_multitier { + mapcache_cache cache; + apr_array_header_t *caches; +}; + + +typedef enum { + MAPCACHE_REST_METHOD_GET, + MAPCACHE_REST_METHOD_HEAD, + MAPCACHE_REST_METHOD_PUT, + MAPCACHE_REST_METHOD_POST, + MAPCACHE_REST_METHOD_DELETE +} mapcache_rest_method; + +typedef enum { + MAPCACHE_REST_PROVIDER_NONE, + MAPCACHE_REST_PROVIDER_S3, + MAPCACHE_REST_PROVIDER_AZURE, + MAPCACHE_REST_PROVIDER_GOOGLE +} mapcache_rest_provider; + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest); +void hmac_sha256(const unsigned char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + unsigned char *mac, unsigned mac_size); +void hmac_sha1(const char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + void *mac); +void sha_hex_encode(unsigned char *sha, unsigned int sha_size); +char *base64_encode(apr_pool_t *pool, const unsigned char *data, size_t input_length); + +typedef struct mapcache_rest_operation mapcache_rest_operation; +struct mapcache_rest_operation { + apr_table_t *headers; + mapcache_rest_method method; + char *tile_url; + void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); +}; + +typedef struct mapcache_rest_configuration mapcache_rest_configuration; +struct mapcache_rest_configuration { + apr_table_t *common_headers; + char *tile_url; + mapcache_rest_operation has_tile; + mapcache_rest_operation get_tile; + mapcache_rest_operation set_tile; + mapcache_rest_operation multi_set_tile; + mapcache_rest_operation delete_tile; + void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); +}; + +/**\class mapcache_cache_rest + * \brief a mapcache_cache on a 3rd party HTTP Rest API + * \implements mapcache_cache + */ +struct mapcache_cache_rest { + mapcache_cache cache; + mapcache_rest_configuration rest; + int use_redirects; + int retry_count; + mapcache_rest_provider provider; +}; + +struct mapcache_cache_s3 { + mapcache_cache_rest cache; + char *id; + char *secret; + char *region; +}; + +struct mapcache_cache_azure { + mapcache_cache_rest cache; + char *id; + char *secret; + char *container; +}; + +struct mapcache_cache_google { + mapcache_cache_rest cache; + char *access; + char *secret; }; #ifdef USE_TIFF @@ -436,6 +588,7 @@ int count_x; int count_y; mapcache_image_format_jpeg *format; + mapcache_locker *locker; }; #endif @@ -451,6 +604,8 @@ char *sql; }; +struct sqlite_conn; + struct mapcache_cache_sqlite { mapcache_cache cache; char *dbfile; @@ -460,9 +615,11 @@ mapcache_cache_sqlite_stmt set_stmt; mapcache_cache_sqlite_stmt delete_stmt; apr_table_t *pragmas; - void (*bind_stmt)(mapcache_context*ctx, void *stmt, mapcache_tile *tile); + void (*bind_stmt)(mapcache_context *ctx, void *stmt, mapcache_cache_sqlite *cache, mapcache_tile *tile); int n_prepared_statements; int detect_blank; + char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; + int count_x, count_y; }; /** @@ -499,9 +656,17 @@ * \brief a mapcache_cache on memcached servers * \implements mapcache_cache */ + +struct mapcache_cache_memcache_server { + char* host; + int port; +}; + struct mapcache_cache_memcache { mapcache_cache cache; - apr_memcache_t *memcache; + int nservers; + struct mapcache_cache_memcache_server *servers; + int detect_blank; }; /** @@ -510,6 +675,49 @@ mapcache_cache* mapcache_cache_memcache_create(mapcache_context *ctx); #endif +#ifdef USE_COUCHBASE +typedef struct mapcache_cache_couchbase mapcache_cache_couchbase; + +/**\class mapcache_cache_couchbase + * \brief a mapcache_cache on couchbase servers + * \implements mapcache_cache + */ +struct mapcache_cache_couchbase { + mapcache_cache cache; +// apr_reslist_t *connection_pool; + char *host; + char *username; + char *password; + char *bucket; + mapcache_context *ctx; +}; + +/** + * \memberof mapcache_cache_couchbase + */ +mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx); +#endif + +#ifdef USE_RIAK +typedef struct mapcache_cache_riak mapcache_cache_riak; + +/**\class mapcache_cache_riak + * \brief a mapcache_cache for riak servers + * \implements mapcache_cache + */ +struct mapcache_cache_riak { + mapcache_cache cache; + char *host; + int port; + RIACK_STRING bucket; +}; + +/** + * \memberof mapcache_cache_riak + */ +mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx); +#endif + /** @} */ @@ -542,8 +750,13 @@ mapcache_service *service; }; +struct mapcache_request_image { + mapcache_request request; + mapcache_image_format *format; +}; + struct mapcache_request_get_tile { - mapcache_request request; + mapcache_request_image image_request; /** * a list of tiles requested by the client @@ -557,8 +770,7 @@ * before being returned to the client */ int ntiles; - mapcache_image_format *format; - + int allow_redirect; }; struct mapcache_http_response { @@ -594,12 +806,11 @@ }; struct mapcache_request_get_map { - mapcache_request request; + mapcache_request_image image_request; mapcache_map **maps; int nmaps; mapcache_getmap_strategy getmap_strategy; mapcache_resample_mode resample_mode; - mapcache_image_format *getmap_format; }; struct mapcache_request_get_capabilities { @@ -647,18 +858,22 @@ mapcache_service *service; }; -struct mapcache_request_proxy { - mapcache_request request; - mapcache_http *http; - apr_table_t *params; - const char *pathinfo; -}; - struct mapcache_forwarding_rule { char *name; mapcache_http *http; apr_array_header_t *match_params; /* actually those are mapcache_dimensions */ int append_pathinfo; + size_t max_post_len; +}; + +struct mapcache_request_proxy { + mapcache_request request; + mapcache_forwarding_rule *rule; + apr_table_t *params; + apr_table_t *headers; + const char *pathinfo; + char *post_buf; + size_t post_len; }; @@ -728,6 +943,7 @@ mapcache_getmap_strategy getmap_strategy; mapcache_resample_mode resample_mode; mapcache_image_format *getmap_format; + int allow_format_override; /* can the client specify which image format should be returned */ }; /**\class mapcache_service_kml @@ -829,7 +1045,7 @@ /** * \brief return the request that corresponds to the given url */ -void mapcache_service_dispatch_request(mapcache_context *ctx, +MS_DLL_EXPORT void mapcache_service_dispatch_request(mapcache_context *ctx, mapcache_request **request, char *pathinfo, apr_table_t *params, @@ -926,25 +1142,22 @@ */ int mapcache_image_has_alpha(mapcache_image *img); +void mapcache_image_fill(mapcache_context *ctx, mapcache_image *image, const unsigned char *fill_color); + /** @} */ /** \defgroup http HTTP Request handling*/ /** @{ */ void mapcache_http_do_request(mapcache_context *ctx, mapcache_http *req, mapcache_buffer *data, apr_table_t *headers, long *http_code); -void mapcache_http_do_request_with_params(mapcache_context *ctx, mapcache_http *req, apr_table_t *params, - mapcache_buffer *data, apr_table_t *headers, long *http_code); char* mapcache_http_build_url(mapcache_context *ctx, char *base, apr_table_t *params); -apr_table_t *mapcache_http_parse_param_string(mapcache_context *ctx, char *args); +MS_DLL_EXPORT apr_table_t *mapcache_http_parse_param_string(mapcache_context *ctx, char *args); /** @} */ /** \defgroup configuration Configuration*/ /** @{ */ -struct mapcache_server_cfg { - apr_hash_t *aliases; /**< list of mapcache configurations aliased to a server uri */ -}; @@ -954,12 +1167,72 @@ MAPCACHE_MODE_MIRROR_SPLIT } mapcache_mode; +typedef enum { + MAPCACHE_LOCKER_DISK, + MAPCACHE_LOCKER_MEMCACHE, + MAPCACHE_LOCKER_FALLBACK +} mapcache_lock_mode; + +typedef enum { + MAPCACHE_LOCK_AQUIRED, + MAPCACHE_LOCK_LOCKED, + MAPCACHE_LOCK_NOENT +} mapcache_lock_result; + + +struct mapcache_locker{ + mapcache_lock_result (*aquire_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock); + mapcache_lock_result (*ping_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock); + void (*release_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock); + + void (*parse_xml)(mapcache_context *ctx, mapcache_locker *self, ezxml_t node); + mapcache_lock_mode type; + double timeout; + double retry_interval; /* time to wait before checking again on a lock, in seconds */ +}; + +typedef struct { + mapcache_locker locker; + + /** + * directory where lock files will be placed. + * Must be readable and writable by the apache user. + * Must be placed on a network mounted shared directory if multiple mapcache instances + * need to be synchronized + */ + const char *dir; +} mapcache_locker_disk; + +mapcache_locker* mapcache_locker_disk_create(mapcache_context *ctx); + +#ifdef USE_MEMCACHE +typedef struct { + char *host; + int port; +} mapcache_locker_memcache_server; + +typedef struct { + mapcache_locker locker; + int nservers; + mapcache_locker_memcache_server *servers; +} mapcache_locker_memcache; + +mapcache_locker* mapcache_locker_memcache_create(mapcache_context *ctx); +#endif + +typedef struct { + mapcache_locker locker; + apr_array_header_t *lockers; +} mapcache_locker_fallback; + +mapcache_locker* mapcache_locker_fallback_create(mapcache_context *ctx); + +void mapcache_config_parse_locker(mapcache_context *ctx, ezxml_t node, mapcache_locker **locker); + /** * a configuration that will be served */ struct mapcache_cfg { - char *configFile; /**< the filename from which this configuration was loaded */ - /** * a list of services that will be responded to */ @@ -1011,26 +1284,10 @@ apr_table_t *metadata; - /** - * directory where lock files will be placed. - * Must be readable and writable by the apache user. - * Must be placed on a network mounted shared directory if multiple mapcache instances - * need to be synchronized - */ - const char *lockdir; - - /** - * time in nanoseconds to wait before rechecking for lockfile presence - */ - apr_interval_time_t lock_retry_interval; /* time in nanoseconds to wait before rechecking for lockfile presence */ + mapcache_locker *locker; int threaded_fetching; - /** - * the uri where the base of the service is mapped - */ - const char *endpoint; - /* for fastcgi only */ int autoreload; /* should the modification time of the config file be recorded and the file be reparsed if it is modified. */ @@ -1051,14 +1308,14 @@ * @param pool * @return */ -void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi); -void mapcache_configuration_post_config(mapcache_context *ctx, mapcache_cfg *config); +MS_DLL_EXPORT void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi); +MS_DLL_EXPORT void mapcache_configuration_post_config(mapcache_context *ctx, mapcache_cfg *config); void mapcache_configuration_parse_xml(mapcache_context *ctx, const char *filename, mapcache_cfg *config); -mapcache_cfg* mapcache_configuration_create(apr_pool_t *pool); +MS_DLL_EXPORT mapcache_cfg* mapcache_configuration_create(apr_pool_t *pool); mapcache_source* mapcache_configuration_get_source(mapcache_cfg *config, const char *key); -mapcache_cache* mapcache_configuration_get_cache(mapcache_cfg *config, const char *key); +MS_DLL_EXPORT mapcache_cache* mapcache_configuration_get_cache(mapcache_cfg *config, const char *key); mapcache_grid *mapcache_configuration_get_grid(mapcache_cfg *config, const char *key); -mapcache_tileset* mapcache_configuration_get_tileset(mapcache_cfg *config, const char *key); +MS_DLL_EXPORT mapcache_tileset* mapcache_configuration_get_tileset(mapcache_cfg *config, const char *key); mapcache_image_format *mapcache_configuration_get_image_format(mapcache_cfg *config, const char *key); void mapcache_configuration_add_image_format(mapcache_cfg *config, mapcache_image_format *format, const char * key); void mapcache_configuration_add_source(mapcache_cfg *config, mapcache_source *source, const char * key); @@ -1096,6 +1353,14 @@ */ mapcache_cache* mapcache_cache_disk_create(mapcache_context *ctx); +/** + * \memberof mapcache_cache_rest + */ +mapcache_cache* mapcache_cache_rest_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_s3_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_azure_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_google_create(mapcache_context *ctx); + #ifdef USE_TIFF /** * \memberof mapcache_cache_tiff @@ -1103,6 +1368,10 @@ mapcache_cache* mapcache_cache_tiff_create(mapcache_context *ctx); #endif +mapcache_cache* mapcache_cache_composite_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_fallback_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_multitier_create(mapcache_context *ctx); + /** \defgroup tileset Tilesets*/ /** @{ */ @@ -1125,6 +1394,8 @@ * \sa mapcache_image_format */ mapcache_buffer *encoded_data; + char *redirect; + int allow_redirect; mapcache_image *raw_image; apr_time_t mtime; /**< last modification time */ int expires; /**< time in seconds after which the tile should be rechecked for validity */ @@ -1168,7 +1439,7 @@ MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT, MAPCACHE_GRID_ORIGIN_TOP_LEFT, MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT, - MAPCACHE_GRID_ORIGIN_TOP_RIGHT, + MAPCACHE_GRID_ORIGIN_TOP_RIGHT } mapcache_grid_origin; struct mapcache_grid { @@ -1200,7 +1471,7 @@ mapcache_extent *restricted_extent; mapcache_extent_i *grid_limits; int minz,maxz; - + /** * tiles above this zoom level will not be stored to the cache, but will be * dynamically generated (either by reconstructing from lower level tiles, or @@ -1209,6 +1480,8 @@ int max_cached_zoom; mapcache_outofzoom_strategy outofzoom_strategy; + + apr_array_header_t *intermediate_grids; }; /**\class mapcache_tileset @@ -1262,7 +1535,7 @@ /** * the cache in which the tiles should be stored */ - mapcache_cache *cache; + mapcache_cache *_cache; /** * the source from which tiles should be requested @@ -1301,7 +1574,8 @@ mapcache_grid_link *grid_link, mapcache_extent *bbox, int width, int height, int *ntiles, - mapcache_tile ***tiles); + mapcache_tile ***tiles, + mapcache_grid_link **effectively_used_grid_link); mapcache_image* mapcache_tileset_assemble_map_tiles(mapcache_context *ctx, mapcache_tileset *tileset, mapcache_grid_link *grid_link, @@ -1338,14 +1612,14 @@ */ void mapcache_tileset_get_level(mapcache_context *ctx, mapcache_tileset *tileset, double *resolution, int *level); -void mapcache_grid_get_closest_level(mapcache_context *ctx, mapcache_grid_link *grid, double resolution, int *level); -void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile); +mapcache_grid_link* mapcache_grid_get_closest_wms_level(mapcache_context *ctx, mapcache_grid_link *grid, double resolution, int *level); +MS_DLL_EXPORT void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile); /** * \brief delete tile from cache * @param whole_metatile delete all the other tiles from the metatile to */ -void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile); +MS_DLL_EXPORT void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile); int mapcache_grid_is_bbox_aligned(mapcache_context *ctx, mapcache_grid *grid, mapcache_extent *bbox); @@ -1356,7 +1630,7 @@ * @param pool * @return */ -mapcache_tile* mapcache_tileset_tile_create(apr_pool_t *pool, mapcache_tileset *tileset, mapcache_grid_link *grid_link); +MS_DLL_EXPORT mapcache_tile* mapcache_tileset_tile_create(apr_pool_t *pool, mapcache_tileset *tileset, mapcache_grid_link *grid_link); mapcache_tile* mapcache_tileset_tile_clone(apr_pool_t *pool, mapcache_tile *src); @@ -1389,27 +1663,27 @@ void mapcache_tileset_add_watermark(mapcache_context *ctx, mapcache_tileset *tileset, const char *filename); -int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, char *resource); -void mapcache_unlock_resource(mapcache_context *ctx, char *resource); +MS_DLL_EXPORT int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void **lock); +MS_DLL_EXPORT void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void *lock); -mapcache_metatile* mapcache_tileset_metatile_get(mapcache_context *ctx, mapcache_tile *tile); -void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt); -char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt); +MS_DLL_EXPORT mapcache_metatile* mapcache_tileset_metatile_get(mapcache_context *ctx, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt); +MS_DLL_EXPORT char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt); /** @} */ -mapcache_http_response* mapcache_core_get_capabilities(mapcache_context *ctx, mapcache_service *service, mapcache_request_get_capabilities *req_caps, char *url, char *path_info, mapcache_cfg *config); -mapcache_http_response* mapcache_core_get_tile(mapcache_context *ctx, mapcache_request_get_tile *req_tile); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_capabilities(mapcache_context *ctx, mapcache_service *service, mapcache_request_get_capabilities *req_caps, char *url, char *path_info, mapcache_cfg *config); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_tile(mapcache_context *ctx, mapcache_request_get_tile *req_tile); -mapcache_http_response* mapcache_core_get_map(mapcache_context *ctx, mapcache_request_get_map *req_map); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_map(mapcache_context *ctx, mapcache_request_get_map *req_map); -mapcache_http_response* mapcache_core_get_featureinfo(mapcache_context *ctx, mapcache_request_get_feature_info *req_fi); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_featureinfo(mapcache_context *ctx, mapcache_request_get_feature_info *req_fi); -mapcache_http_response* mapcache_core_proxy_request(mapcache_context *ctx, mapcache_request_proxy *req_proxy); -mapcache_http_response* mapcache_core_respond_to_error(mapcache_context *ctx); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_proxy_request(mapcache_context *ctx, mapcache_request_proxy *req_proxy); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_respond_to_error(mapcache_context *ctx); /* in grid.c */ @@ -1418,7 +1692,7 @@ const char* mapcache_grid_get_crs(mapcache_context *ctx, mapcache_grid *grid); const char* mapcache_grid_get_srs(mapcache_context *ctx, mapcache_grid *grid); -void mapcache_grid_get_extent(mapcache_context *ctx, mapcache_grid *grid, +MS_DLL_EXPORT void mapcache_grid_get_extent(mapcache_context *ctx, mapcache_grid *grid, int x, int y, int z, mapcache_extent *bbox); /** * \brief compute x y value for given lon/lat (dx/dy) and given zoomlevel @@ -1430,7 +1704,7 @@ * @param x * @param y */ -void mapcache_grid_get_xy(mapcache_context *ctx, mapcache_grid *grid, double dx, double dy, int z, int *x, int *y); +MS_DLL_EXPORT void mapcache_grid_get_xy(mapcache_context *ctx, mapcache_grid *grid, double dx, double dy, int z, int *x, int *y); double mapcache_grid_get_resolution(mapcache_extent *bbox, int sx, int sy); double mapcache_grid_get_horizontal_resolution(mapcache_extent *bbox, int width); @@ -1450,12 +1724,12 @@ * \param extent * \param tolerance the number of tiles around the given extent that can be requested without returning an error. */ -void mapcache_grid_compute_limits(const mapcache_grid *grid, const mapcache_extent *extent, mapcache_extent_i *limits, int tolerance); +MS_DLL_EXPORT void mapcache_grid_compute_limits(const mapcache_grid *grid, const mapcache_extent *extent, mapcache_extent_i *limits, int tolerance); /* in util.c */ -int mapcache_util_extract_int_list(mapcache_context *ctx, const char* args, const char *sep, int **numbers, +MS_DLL_EXPORT int mapcache_util_extract_int_list(mapcache_context *ctx, const char* args, const char *sep, int **numbers, int *numbers_count); -int mapcache_util_extract_double_list(mapcache_context *ctx, const char* args, const char *sep, double **numbers, +MS_DLL_EXPORT int mapcache_util_extract_double_list(mapcache_context *ctx, const char* args, const char *sep, double **numbers, int *numbers_count); char *mapcache_util_str_replace(apr_pool_t *pool, const char *string, const char *substr, const char *replacement ); @@ -1474,6 +1748,7 @@ char* mapcache_util_get_tile_key(mapcache_context *ctx, mapcache_tile *tile, char *stemplate, char* sanitized_chars, char *sanitize_to); +void mapcache_make_parent_dirs(mapcache_context *ctx, char *filename); /**\defgroup imageio Image IO */ /** @{ */ @@ -1484,6 +1759,7 @@ typedef enum { MAPCACHE_COMPRESSION_BEST, /**< best but slowest compression*/ MAPCACHE_COMPRESSION_FAST, /**< fast compression*/ + MAPCACHE_COMPRESSION_DISABLE, /**< no compression*/ MAPCACHE_COMPRESSION_DEFAULT /**< default compression*/ } mapcache_compression_type; @@ -1535,7 +1811,7 @@ mapcache_image_format *opaque; }; -mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, const unsigned char *hex_color, int *is_empty); +mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, int width, int height, const unsigned char *hex_color, int *is_empty); mapcache_image_format* mapcache_imageio_create_mixed_format(apr_pool_t *pool, @@ -1658,7 +1934,8 @@ MAPCACHE_DIMENSION_VALUES, MAPCACHE_DIMENSION_REGEX, MAPCACHE_DIMENSION_INTERVALS, - MAPCACHE_DIMENSION_TIME + MAPCACHE_DIMENSION_TIME, + MAPCACHE_DIMENSION_SQLITE } mapcache_dimension_type; struct mapcache_dimension { @@ -1667,6 +1944,7 @@ char *unit; apr_table_t *metadata; char *default_value; + int skip_validation; /** * \brief validate the given value @@ -1683,7 +1961,7 @@ * * \returns a list of character strings that will be included in the capabilities element */ - const char** (*print_ogc_formatted_values)(mapcache_context *context, mapcache_dimension *dimension); + apr_array_header_t* (*print_ogc_formatted_values)(mapcache_context *context, mapcache_dimension *dimension); /** * \brief parse the value given in the configuration @@ -1698,6 +1976,13 @@ int case_sensitive; }; +struct mapcache_dimension_sqlite { + mapcache_dimension dimension; + char *dbfile; + char *validate_query; + char *list_query; +}; + struct mapcache_dimension_regex { mapcache_dimension dimension; char *regex_string; @@ -1721,6 +2006,7 @@ }; mapcache_dimension* mapcache_dimension_values_create(apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_sqlite_create(apr_pool_t *pool); mapcache_dimension* mapcache_dimension_regex_create(apr_pool_t *pool); mapcache_dimension* mapcache_dimension_intervals_create(apr_pool_t *pool); mapcache_dimension* mapcache_dimension_time_create(apr_pool_t *pool); @@ -1734,13 +2020,13 @@ MAPCACHE_TIMEDIMENSION_SOURCE_SQLITE } mapcache_timedimension_source_type; -apr_array_header_t* mapcache_timedimension_get_entries_for_value(mapcache_context *ctx, mapcache_timedimension *timedimesnion, +MS_DLL_EXPORT apr_array_header_t* mapcache_timedimension_get_entries_for_value(mapcache_context *ctx, mapcache_timedimension *timedimesnion, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, const char *value); struct mapcache_timedimension { mapcache_timedimension_assembly_type assembly_type; void (*configuration_parse_xml)(mapcache_context *context, mapcache_timedimension *dim, ezxml_t node); - apr_array_header_t* (*get_entries_for_interval)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset, + apr_array_header_t* (*get_entries_for_interval)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end); apr_array_header_t* (*get_all_entries)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset); char *default_value; @@ -1759,6 +2045,26 @@ int mapcache_is_axis_inverted(const char *srs); +typedef struct mapcache_pooled_connection_container mapcache_pooled_connection_container; +typedef struct mapcache_pooled_connection mapcache_pooled_connection; +typedef struct mapcache_pooled_connection_private_data mapcache_pooled_connection_private_data; + +struct mapcache_pooled_connection { + mapcache_pooled_connection_private_data *private; + void *connection; +}; + +typedef void (*mapcache_connection_constructor)(mapcache_context *ctx, void **connection, void *params); +typedef void (*mapcache_connection_destructor)(void *connection); + +MS_DLL_EXPORT apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool); +mapcache_pooled_connection* mapcache_connection_pool_get_connection(mapcache_context *ctx, char *key, + mapcache_connection_constructor constructor, + mapcache_connection_destructor destructor, + void *params); +void mapcache_connection_pool_invalidate_connection(mapcache_context *ctx, mapcache_pooled_connection *connection); +void mapcache_connection_pool_release_connection(mapcache_context *ctx, mapcache_pooled_connection *connection); + #endif /* MAPCACHE_H_ */ /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/include/util.h mapcache-1.4.1/include/util.h --- mapcache-1.2.1/include/util.h 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/include/util.h 2016-02-25 15:47:16.000000000 +0000 @@ -87,13 +87,19 @@ #endif #endif +#if defined(_WIN32) && !defined(__CYGWIN__) +# define MS_DLL_EXPORT __declspec(dllexport) +#else +#define MS_DLL_EXPORT +#endif #if defined(_WIN32) struct mctimeval { long tv_sec; /* seconds */ long tv_usec; /* and microseconds */ }; -void mapcache_gettimeofday(struct mctimeval *t, void *__not_used_here__); +MS_DLL_EXPORT void mapcache_gettimeofday(struct mctimeval *t, void *__not_used_here__); +MS_DLL_EXPORT char * strptime (const char *buf, const char *format, struct tm *timeptr); #else # include /* for gettimeofday() */ # define mctimeval timeval diff -Nru mapcache-1.2.1/lib/buffer.c mapcache-1.4.1/lib/buffer.c --- mapcache-1.2.1/lib/buffer.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/buffer.c 2016-02-25 15:47:16.000000000 +0000 @@ -57,6 +57,7 @@ mapcache_buffer *buffer = apr_pcalloc(pool, sizeof(mapcache_buffer)); if(!buffer) return NULL; buffer->pool = pool; + if(initialStorage <=0) initialStorage = 1; buffer->avail = initialStorage; if(buffer->avail) { buffer->buf = malloc(buffer->avail); diff -Nru mapcache-1.2.1/lib/cache_bdb.c mapcache-1.4.1/lib/cache_bdb.c --- mapcache-1.2.1/lib/cache_bdb.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/cache_bdb.c 2016-02-25 15:47:16.000000000 +0000 @@ -51,9 +51,6 @@ #define PAGESIZE 64*1024 #define CACHESIZE 1024*1024 -static apr_hash_t *ro_connection_pools = NULL; -static apr_hash_t *rw_connection_pools = NULL; - struct bdb_env { DB* db; DB_ENV *env; @@ -61,168 +58,100 @@ char *errmsg; }; -static apr_status_t _bdb_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) +void mapcache_bdb_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; int env_flags; int mode; mapcache_cache_bdb *cache = (mapcache_cache_bdb*)params; - char *dbfile = apr_pstrcat(pool,cache->basedir,"/",cache->cache.name,".db",NULL); + char *dbfile = malloc(strlen(cache->basedir)+strlen(cache->cache.name)+strlen("/.db")+1); struct bdb_env *benv = calloc(1,sizeof(struct bdb_env)); + dbfile[0]=0; + strcat(strcat(strcat(strcat(dbfile,cache->basedir),"/"),cache->cache.name),".db"); /*yuk yuk*/ *conn_ = benv; ret = db_env_create(&benv->env, 0); if(ret) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for db_env_create: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx, 500, "bdb cache failure for db_env_create: %s", db_strerror(ret)); + goto cleanup_error; } ret = benv->env->set_cachesize(benv->env,0,CACHESIZE,1); /* set a larger cache size than default */ if(ret) { - benv->errmsg = apr_psprintf(pool, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx, 500, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret)); + goto cleanup_error; } env_flags = DB_INIT_CDB|DB_INIT_MPOOL|DB_CREATE; ret = benv->env->open(benv->env,cache->basedir,env_flags,0); if(ret) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for env->open: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure for env->open: %s", db_strerror(ret)); + goto cleanup_error; } if ((ret = db_create(&benv->db, benv->env, 0)) != 0) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for db_create: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure for db_create: %s", db_strerror(ret)); + goto cleanup_error; } mode = DB_BTREE; ret = benv->db->set_pagesize(benv->db,PAGESIZE); /* set pagesize to maximum allowed, as tile data is usually pretty large */ if(ret) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret)); + goto cleanup_error; } if ((ret = benv->db->open(benv->db, NULL, dbfile, NULL, mode, DB_CREATE, 0664)) != 0) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure 1 for db->open: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure 1 for db->open: %s", db_strerror(ret)); + goto cleanup_error; } - return APR_SUCCESS; + + goto cleanup; + +cleanup_error: + free(benv); +cleanup: + free(dbfile); } -static apr_status_t _bdb_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) +void mapcache_bdb_connection_destructor(void *conn_) { struct bdb_env *benv = (struct bdb_env*)conn_; benv->db->close(benv->db,0); benv->env->close(benv->env,0); free(benv); - - return APR_SUCCESS; } -static struct bdb_env* _bdb_get_conn(mapcache_context *ctx, mapcache_tile* tile, int readonly) { - apr_status_t rv; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; - +static mapcache_pooled_connection* _bdb_get_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile* tile, int readonly) { struct bdb_env *benv; - apr_hash_t *pool_container; - apr_reslist_t *pool = NULL; - if(readonly) { - pool_container = ro_connection_pools; - } else { - pool_container = rw_connection_pools; - } - if(!pool_container || NULL == (pool = apr_hash_get(pool_container,cache->cache.name, APR_HASH_KEY_STRING)) ) { -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(!ro_connection_pools) { - ro_connection_pools = apr_hash_make(ctx->process_pool); - rw_connection_pools = apr_hash_make(ctx->process_pool); - } - - /* probably doesn't exist, unless the previous mutex locked us, so we check */ - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - if(!pool) { - /* there where no existing connection pools, create them*/ - rv = apr_reslist_create(&pool, - 0 /* min */, - 10 /* soft max */, - 200 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _bdb_reslist_get_connection, /* resource constructor */ - _bdb_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb ro connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(ro_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - rv = apr_reslist_create(&pool, - 0 /* min */, - 1 /* soft max */, - 1 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _bdb_reslist_get_connection, /* resource constructor */ - _bdb_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb rw connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(rw_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - } -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(readonly) - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - else - pool = apr_hash_get(rw_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - assert(pool); - } - rv = apr_reslist_acquire(pool, (void **)&benv); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to aquire connection to bdb backend: %s", (benv&& benv->errmsg)?benv->errmsg:"unknown error"); - return NULL; - } + mapcache_pooled_connection *pc; + char *conn_key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",cache->cache.name,NULL); + pc = mapcache_connection_pool_get_connection(ctx,conn_key,mapcache_bdb_connection_constructor, mapcache_bdb_connection_destructor, cache); + if(GC_HAS_ERROR(ctx)) return NULL; + benv = pc->connection; benv->readonly = readonly; - return benv; + return pc; } -static void _bdb_release_conn(mapcache_context *ctx, mapcache_tile *tile, struct bdb_env *benv) +static void _bdb_release_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile *tile, mapcache_pooled_connection *pc) { - apr_reslist_t *pool; - apr_hash_t *pool_container; - if(benv->readonly) { - pool_container = ro_connection_pools; - } else { - pool_container = rw_connection_pools; - } - pool = apr_hash_get(pool_container,tile->tileset->cache->name, APR_HASH_KEY_STRING); if(GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(pool,(void*)benv); + mapcache_connection_pool_invalidate_connection(ctx, pc); } else { - apr_reslist_release(pool, (void*)benv); + mapcache_connection_pool_release_connection(ctx,pc); } } -static int _mapcache_cache_bdb_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_bdb_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { int ret; DBT key; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - struct bdb_env *benv = _bdb_get_conn(ctx,tile,1); + mapcache_pooled_connection *pc; + struct bdb_env *benv; + pc = _bdb_get_conn(ctx,cache,tile,1); if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; + benv = pc->connection; memset(&key, 0, sizeof(DBT)); key.data = skey; key.size = strlen(skey)+1; @@ -237,18 +166,21 @@ ctx->set_error(ctx,500,"bdb backend failure on tile_exists: %s",db_strerror(ret)); ret= MAPCACHE_FALSE; } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); return ret; } -static void _mapcache_cache_bdb_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_bdb_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { DBT key; int ret; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - struct bdb_env *benv = _bdb_get_conn(ctx,tile,0); + mapcache_pooled_connection *pc; + struct bdb_env *benv; + pc = _bdb_get_conn(ctx,cache,tile,0); GC_CHECK_ERROR(ctx); + benv = pc->connection; memset(&key, 0, sizeof(DBT)); key.data = skey; key.size = strlen(skey)+1; @@ -260,17 +192,21 @@ if(ret) ctx->set_error(ctx,500,"bdb backend sync failure on tile_delete: %s",db_strerror(ret)); } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); } -static int _mapcache_cache_bdb_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_bdb_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { DBT key,data; int ret; - struct bdb_env *benv = _bdb_get_conn(ctx,tile,1); - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; - char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - if(GC_HAS_ERROR(ctx)) return MAPCACHE_FAILURE; + char *skey; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; + mapcache_pooled_connection *pc; + struct bdb_env *benv; + pc = _bdb_get_conn(ctx,cache,tile,1); + if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; + benv = pc->connection; + skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); data.flags = DB_DBT_MALLOC; @@ -282,7 +218,7 @@ if(ret == 0) { if(((char*)(data.data))[0] == '#') { - tile->encoded_data = mapcache_empty_png_decode(ctx,(unsigned char*)data.data,&tile->nodata); + tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy, (unsigned char*)data.data,&tile->nodata); } else { tile->encoded_data = mapcache_buffer_create(0,ctx->pool); tile->encoded_data->buf = data.data; @@ -298,20 +234,20 @@ ctx->set_error(ctx,500,"bdb backend failure on tile_get: %s",db_strerror(ret)); ret = MAPCACHE_FAILURE; } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); return ret; } -static void _mapcache_cache_bdb_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_bdb_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { DBT key,data; int ret; apr_time_t now; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - struct bdb_env *benv = _bdb_get_conn(ctx,tile,0); - GC_CHECK_ERROR(ctx); + mapcache_pooled_connection *pc; + struct bdb_env *benv; now = apr_time_now(); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); @@ -323,6 +259,7 @@ tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); } + if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { data.size = 5+sizeof(apr_time_t); data.data = apr_palloc(ctx->pool,data.size); @@ -340,6 +277,11 @@ tile->encoded_data->size -= sizeof(apr_time_t); } + pc = _bdb_get_conn(ctx,cache,tile,0); + GC_CHECK_ERROR(ctx); + benv = pc->connection; + + ret = benv->db->put(benv->db,NULL,&key,&data,0); if(ret != 0) { ctx->set_error(ctx,500,"dbd backend failed on tile_set: %s", db_strerror(ret)); @@ -348,21 +290,25 @@ if(ret) ctx->set_error(ctx,500,"bdb backend sync failure on tile_set: %s",db_strerror(ret)); } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); } -static void _mapcache_cache_bdb_multiset(mapcache_context *ctx, mapcache_tile *tiles, int ntiles) +static void _mapcache_cache_bdb_multiset(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { DBT key,data; int ret,i; apr_time_t now; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tiles[0].tileset->cache; - struct bdb_env *benv = _bdb_get_conn(ctx,&tiles[0],0); - GC_CHECK_ERROR(ctx); + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; + mapcache_pooled_connection *pc; + struct bdb_env *benv; now = apr_time_now(); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); + pc = _bdb_get_conn(ctx,cache,&tiles[0],0); + GC_CHECK_ERROR(ctx); + benv = pc->connection; + for(i=0; ikey_template,NULL,NULL); if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); - GC_CHECK_ERROR(ctx); + if(GC_HAS_ERROR(ctx)) { + _bdb_release_conn(ctx,cache,&tiles[0],pc); + return; + } } if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { data.size = 5+sizeof(apr_time_t); @@ -383,7 +332,10 @@ } else { if(!tile->encoded_data) { tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); - GC_CHECK_ERROR(ctx); + if(GC_HAS_ERROR(ctx)) { + _bdb_release_conn(ctx,cache,&tiles[0],pc); + return; + } } mapcache_buffer_append(tile->encoded_data,sizeof(apr_time_t),&now); data.data = tile->encoded_data->buf; @@ -404,7 +356,7 @@ if(ret) ctx->set_error(ctx,500,"bdb backend sync failure on sync in tile_multiset: %s",db_strerror(ret)); } - _bdb_release_conn(ctx,&tiles[0],benv); + _bdb_release_conn(ctx,cache,&tiles[0],pc); } diff -Nru mapcache-1.2.1/lib/cache_composite.c mapcache-1.4.1/lib/cache_composite.c --- mapcache-1.2.1/lib/cache_composite.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/cache_composite.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,205 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: composite cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without compositeriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" + +static mapcache_cache_composite_cache_link* _mapcache_cache_link_create(apr_pool_t *pool) { + mapcache_cache_composite_cache_link *cl = apr_pcalloc(pool, sizeof(mapcache_cache_composite_cache_link)); + cl->cache=NULL; + cl->dimensions=NULL; + cl->grids=NULL; + cl->maxzoom=-1; + cl->minzoom=-1; + return cl; +} +/** + * returns the mapcache_cache to use for a given tile + * @param ctx + * @param tile + * @return + */ +static mapcache_cache* _mapcache_composite_cache_get(mapcache_context *ctx, mapcache_cache_composite *cache, mapcache_tile *tile) { + int i; + for(i=0; icache_links->nelts; i++) { + mapcache_cache_composite_cache_link *cache_link = APR_ARRAY_IDX(cache->cache_links,i,mapcache_cache_composite_cache_link*); + if(cache_link->minzoom != -1 && tile->z < cache_link->minzoom) continue; + if(cache_link->maxzoom != -1 && tile->z > cache_link->maxzoom) continue; + if(cache_link->grids) { + int j; + for(j=0;jgrids->nelts;j++) { + char *grid_name = APR_ARRAY_IDX(cache_link->grids,j,char*); + if(!strcmp(tile->grid_link->grid->name,grid_name)) + break; + } + /* not found */ + if(j == cache_link->grids->nelts) continue; + } + return cache_link->cache; + } + ctx->set_error(ctx, 500, "no cache matches for given tile request"); + return NULL; +} +static int _mapcache_cache_composite_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + if(GC_HAS_ERROR(ctx) || !subcache) + return MAPCACHE_FAILURE; + return subcache->tile_exists(ctx, subcache, tile); +} + +static void _mapcache_cache_composite_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + /*delete the tile itself*/ + subcache->tile_delete(ctx,subcache,tile); +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the composite server + * \private \memberof mapcache_cache_composite + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_composite_tile_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + GC_CHECK_ERROR_RETURN(ctx); + return subcache->tile_get(ctx,subcache,tile); +} + +static void _mapcache_cache_composite_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + return subcache->tile_set(ctx,subcache,tile); +} + +static void _mapcache_cache_composite_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, &tiles[0]); + GC_CHECK_ERROR(ctx); + if(subcache->tile_multi_set) { + return subcache->tile_multi_set(ctx,subcache,tiles,ntiles); + } else { + int i; + for(i=0; itile_set(ctx, subcache, &tiles[i]); + } + } +} + +/** + * \private \memberof mapcache_cache_composite + */ +static void _mapcache_cache_composite_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + cache->cache_links = apr_array_make(ctx->pool,3,sizeof(mapcache_cache_composite_cache_link*)); + for(cur_node = ezxml_child(node,"cache"); cur_node; cur_node = cur_node->next) { + char *sZoom; + int zoom; + mapcache_cache *refcache = mapcache_configuration_get_cache(config, cur_node->txt); + mapcache_cache_composite_cache_link *cachelink; + if(!refcache) { + ctx->set_error(ctx, 400, "composite cache \"%s\" references cache \"%s\"," + " but it is not configured (hint:referenced caches must be declared before this composite cache in the xml file)", pcache->name, cur_node->txt); + return; + } + cachelink = _mapcache_cache_link_create(ctx->pool); + cachelink->cache = refcache; + + sZoom = (char*)ezxml_attr(cur_node,"max-zoom"); + if(sZoom) { + char *endptr; + zoom = (int)strtol(sZoom,&endptr,10); + if(*endptr != 0 || zoom < 0) { + ctx->set_error(ctx, 400, "failed to parse cache max-zoom %s (expecting a positive integer)", + sZoom); + return; + } + cachelink->maxzoom = zoom; + } + sZoom = (char*)ezxml_attr(cur_node,"min-zoom"); + if(sZoom) { + char *endptr; + zoom = (int)strtol(sZoom,&endptr,10); + if(*endptr != 0 || zoom < 0) { + ctx->set_error(ctx, 400, "failed to parse cache min-zoom %s (expecting a positive integer)", + sZoom); + return; + } + cachelink->minzoom = zoom; + } + + APR_ARRAY_PUSH(cache->cache_links,mapcache_cache_composite_cache_link*) = cachelink; + } +} + +/** + * \private \memberof mapcache_cache_composite + */ +static void _mapcache_cache_composite_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ +} + + +/** + * \brief creates and initializes a mapcache_cache_composite + */ +mapcache_cache* mapcache_cache_composite_create(mapcache_context *ctx) +{ + mapcache_cache_composite *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_composite)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate composite cache"); + return NULL; + } + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_COMPOSITE; + cache->cache.tile_delete = _mapcache_cache_composite_tile_delete; + cache->cache.tile_get = _mapcache_cache_composite_tile_get; + cache->cache.tile_exists = _mapcache_cache_composite_tile_exists; + cache->cache.tile_set = _mapcache_cache_composite_tile_set; + cache->cache.tile_multi_set = _mapcache_cache_composite_tile_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_composite_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_composite_configuration_parse_xml; + return (mapcache_cache*)cache; +} diff -Nru mapcache-1.2.1/lib/cache_couchbase.c mapcache-1.4.1/lib/cache_couchbase.c --- mapcache-1.2.1/lib/cache_couchbase.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/cache_couchbase.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,462 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching support file: couchbase cache backend. + * Author: Michael Downey and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache-config.h" +#ifdef USE_COUCHBASE + +#include "mapcache.h" +#include +#include +#include + +typedef struct getStruct +{ + mapcache_buffer *tileBuffer; + libcouchbase_error_t error; +} getStruct_t; + +/* Not sure if we need this. */ +static void _couchbase_error_callback(libcouchbase_t instance, + libcouchbase_error_t error, + const char *errinfo) +{ + /* Ignore timeouts... */ + if (error != LIBCOUCHBASE_ETIMEDOUT) { + fprintf(stderr, "\nFATAL ERROR: %s\n", + libcouchbase_strerror(instance, error)); + if (errinfo && strlen(errinfo) != 0) { + fprintf(stderr, "\t\"%s\"\n", errinfo); + } + } +} + +static void _couchbase_get_callback(libcouchbase_t instance, + const void *cookie, + libcouchbase_error_t error, + const void *key, libcouchbase_size_t nkey, + const void *bytes, libcouchbase_size_t nbytes, + libcouchbase_uint32_t flags, libcouchbase_cas_t cas) +{ + (void)instance; + (void)key; + (void)nkey; + (void)flags; + (void)cas; + + if (cookie) + { + getStruct_t *request = (getStruct_t*)cookie; + + request->error = error; + + if (error == LIBCOUCHBASE_SUCCESS && request->tileBuffer) + { + mapcache_buffer_append(request->tileBuffer, nbytes, (void*)bytes); + } + } +} + +static void _couchbase_store_callback(libcouchbase_t instance, + const void* cookie, + libcouchbase_storage_t unknown, + libcouchbase_error_t error, + const void* unknown2, + libcouchbase_size_t unknown3, + libcouchbase_cas_t cas) +{ + (void)instance; + (void)unknown; + (void)unknown2; + (void)unknown3; + (void)cas; + + libcouchbase_error_t* userError = (libcouchbase_error_t*)cookie; + + *userError = error; +} + +static apr_status_t _couchbase_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) { + mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)params; + + libcouchbase_t *instance = apr_pcalloc(pool,sizeof(libcouchbase_t)); + const char *host = cache->host; + const char *username = cache->username; + const char *passwd = cache->password; + const char *bucket = "default"; + + *instance = libcouchbase_create(host, username, passwd, bucket, NULL); + if (*instance == NULL) { + return APR_EGENERAL; + } + + libcouchbase_set_error_callback(*instance, _couchbase_error_callback); + libcouchbase_set_get_callback(*instance, _couchbase_get_callback); + libcouchbase_set_storage_callback(*instance, _couchbase_store_callback); + + if (libcouchbase_connect(*instance) != LIBCOUCHBASE_SUCCESS) { + return APR_EGENERAL; + } + + /* Wait for the connect to compelete */ + libcouchbase_wait(*instance); + + *conn_ = instance; + return APR_SUCCESS; +} + +static apr_status_t _couchbase_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) { + libcouchbase_t *instance = (libcouchbase_t*)conn_; + libcouchbase_destroy(*instance); + return APR_SUCCESS; +} + +static libcouchbase_t* _couchbase_get_connection(mapcache_context *ctx, mapcache_tile *tile) +{ + apr_status_t rv; + libcouchbase_t *instance; + mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache; + + rv = apr_reslist_acquire(cache->connection_pool, (void **)&instance); + if(rv != APR_SUCCESS) { + ctx->set_error(ctx, 500, "failed to aquire connection to couchbase backend: %s", ctx->get_error_message(ctx)); + return NULL; + } + + return instance; +} + +static void _couchbase_release_connection(mapcache_tile *tile, libcouchbase_t* instance) +{ + mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache; + apr_reslist_release(cache->connection_pool, (void*)instance); +} + +static void _couchbase_invalidate_connection(mapcache_tile *tile, libcouchbase_t* instance) +{ + mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache; + apr_reslist_invalidate(cache->connection_pool, (void*)instance); +} + +static int _mapcache_cache_couchbase_has_tile(mapcache_context *ctx, mapcache_tile *tile) { + char *key[1]; + libcouchbase_t *instance; + libcouchbase_error_t error; + size_t keySize[1]; + getStruct_t request; + + key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_FALSE; + } + + keySize[0] = strlen(key[0]); + + instance = _couchbase_get_connection(ctx, tile); + request.tileBuffer = 0; + request.error = LIBCOUCHBASE_KEY_ENOENT; + error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, 0); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error)); + _couchbase_invalidate_connection(tile, instance); + return MAPCACHE_FALSE; + } + + libcouchbase_wait(*instance); + + error = request.error; + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error)); + _couchbase_invalidate_connection(tile, instance); + return MAPCACHE_FALSE; + } + + _couchbase_release_connection(tile, instance); + return MAPCACHE_TRUE; +} + +static void _mapcache_cache_couchbase_delete(mapcache_context *ctx, mapcache_tile *tile) { + char *key; + libcouchbase_t *instance; + libcouchbase_error_t error; + + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + GC_CHECK_ERROR(ctx); + + instance = _couchbase_get_connection(ctx, tile); + + error = libcouchbase_remove(*instance, 0, key, strlen(key), 0); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error)); + } + + libcouchbase_wait(*instance); + + error = libcouchbase_get_last_error(*instance); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error)); + } + + _couchbase_release_connection(tile, instance); +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the couchbase server + * \private \memberof mapcache_cache_couchbase + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_couchbase_get(mapcache_context *ctx, mapcache_tile *tile) { + char *key[1]; + size_t keySize[1]; + libcouchbase_t *instance; + libcouchbase_error_t error; + getStruct_t request; + + key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + + keySize[0] = strlen(key[0]); + + tile->encoded_data = mapcache_buffer_create(0, ctx->pool); + + libcouchbase_time_t expires = 86400; + if(tile->tileset->auto_expire) + expires = tile->tileset->auto_expire; + + instance = _couchbase_get_connection(ctx, tile); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + + request.tileBuffer = tile->encoded_data; + error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, &expires); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase cache returned error on mget %s", libcouchbase_strerror(*instance, error)); + _couchbase_invalidate_connection(tile, instance); + return MAPCACHE_FAILURE; + } + + libcouchbase_wait(*instance); + + if(request.error != LIBCOUCHBASE_SUCCESS) { + _couchbase_release_connection(tile, instance); + return MAPCACHE_CACHE_MISS; + } + + if (tile->encoded_data->size == 0) { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "couchbase cache returned 0-length data for tile %d %d %d", tile->x, tile->y, tile->z); + return MAPCACHE_FAILURE; + } + + apr_time_t now = apr_time_now(); + tile->mtime = now; + + _couchbase_release_connection(tile, instance); + return MAPCACHE_SUCCESS; +} + +/** + * \brief push tile data to couchbase + * + * writes the content of mapcache_tile::data to the configured couchbased instance(s) + * \private \memberof mapcache_cache_couchbase + * \sa mapcache_cache::tile_set() + */ +static void _mapcache_cache_couchbase_set(mapcache_context *ctx, mapcache_tile *tile) { + char *key; + libcouchbase_t *instance; + libcouchbase_error_t error; + const int max_retries = 3; + int retries = max_retries; + apr_interval_time_t delay; + + /* set expiration to one day if not configured */ + libcouchbase_time_t expires = 86400; + if(tile->tileset->auto_expire) + expires = tile->tileset->auto_expire; + + mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache; + key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + GC_CHECK_ERROR(ctx); + + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + + instance = _couchbase_get_connection(ctx, tile); + GC_CHECK_ERROR(ctx); + + do + { + error = libcouchbase_store(*instance, &error, LIBCOUCHBASE_SET, key, strlen(key), tile->encoded_data->buf, tile->encoded_data->size, 0, expires, 0); + if (error != LIBCOUCHBASE_SUCCESS) { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to eror %s.", + tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error)); + return; + } + + libcouchbase_wait(*instance); + + if (error == LIBCOUCHBASE_ETMPFAIL) { + if (retries > 0) { + delay = 100000 * (1 << (max_retries - retries)); // Do an exponential back off of starting at 100 milliseconds + apr_sleep(delay); + } + else { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to %s. Maximum number of retries used.", + tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error)); + return; + } + + --retries; + } + + else if (error != LIBCOUCHBASE_SUCCESS) { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to error %s.", + tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error)); + return; + } + } + while (error == LIBCOUCHBASE_ETMPFAIL); + + _couchbase_release_connection(tile, instance); +} + +/** + * \private \memberof mapcache_cache_couchbase + */ +static void _mapcache_cache_couchbase_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { + ezxml_t cur_node; + apr_status_t rv; + mapcache_cache_couchbase *dcache = (mapcache_cache_couchbase*)cache; + int servercount = 0; + + for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { + servercount++; + } + + if(!servercount) { + ctx->set_error(ctx, 400, "couchbase cache %s has no s configured", cache->name); + return; + } + + if(servercount > 1) { + ctx->set_error(ctx, 400, "couchbase cache %s has more than 1 server configured", cache->name); + return; + } + + cur_node = ezxml_child(node, "server"); + ezxml_t xhost = ezxml_child(cur_node, "host"); /* Host should contain server:port */ + ezxml_t xusername = ezxml_child(cur_node, "username"); + ezxml_t xpasswd = ezxml_child(cur_node, "password"); + ezxml_t xbucket = ezxml_child(cur_node, "bucket"); + + if(!xhost || !xhost->txt || ! *xhost->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->host = apr_pstrdup(ctx->pool, xhost->txt); + if (dcache->host == NULL) { + ctx->set_error(ctx, 400, "cache %s: failed to allocate host string!", cache->name); + return; + } + } + + if(xusername && xusername->txt && *xusername->txt) { + dcache->username = apr_pstrdup(ctx->pool, xusername->txt); + } + + if(xpasswd && xpasswd->txt && *xpasswd->txt) { + dcache->password = apr_pstrdup(ctx->pool, xpasswd->txt); + } + + if(xbucket && xbucket->txt && *xbucket->txt) { + dcache->bucket = apr_pstrdup(ctx->pool, xbucket->txt); + } + + dcache->ctx = ctx; + + rv = apr_reslist_create(&(dcache->connection_pool), + 0 /* min */, + 10 /* soft max */, + 200 /* hard max */, + 60*1000000 /*60 seconds, ttl*/, + _couchbase_reslist_get_connection, /* resource constructor */ + _couchbase_reslist_free_connection, /* resource destructor */ + dcache, ctx->pool); + if(rv != APR_SUCCESS) { + ctx->set_error(ctx, 500, "failed to create couchbase connection pool"); + return; + } +} + +/** + * \private \memberof mapcache_cache_couchbase + */ +static void _mapcache_cache_couchbase_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, mapcache_cfg *cfg) { +} + +/** + * \brief creates and initializes a mapcache_couchbase_cache + */ +mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) { + mapcache_cache_couchbase *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_couchbase)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate couchbase cache"); + return NULL; + } + + cache->cache.metadata = apr_table_make(ctx->pool, 3); + cache->cache.type = MAPCACHE_CACHE_COUCHBASE; + cache->cache.tile_get = _mapcache_cache_couchbase_get; + cache->cache.tile_exists = _mapcache_cache_couchbase_has_tile; + cache->cache.tile_set = _mapcache_cache_couchbase_set; + cache->cache.tile_delete = _mapcache_cache_couchbase_delete; + cache->cache.configuration_parse_xml = _mapcache_cache_couchbase_configuration_parse_xml; + cache->cache.configuration_post_config = _mapcache_cache_couchbase_configuration_post_config; + cache->host = NULL; + cache->username = NULL; + cache->password = NULL; + cache->bucket = NULL; + + return (mapcache_cache*)cache; +} + +#endif + +/* vim: ai ts=3 sts=3 et sw=3 +*/ diff -Nru mapcache-1.2.1/lib/cache_disk.c mapcache-1.4.1/lib/cache_disk.c --- mapcache-1.2.1/lib/cache_disk.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/cache_disk.c 2016-02-25 15:47:16.000000000 +0000 @@ -111,10 +111,10 @@ * \param path pointer to a char* that will contain the filename * \private \memberof mapcache_cache_disk */ -static void _mapcache_cache_disk_base_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_base_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { *path = apr_pstrcat(ctx->pool, - ((mapcache_cache_disk*)tile->tileset->cache)->base_directory,"/", + cache->base_directory,"/", tile->tileset->name,"/", tile->grid_link->grid->name, NULL); @@ -129,11 +129,11 @@ } } -static void _mapcache_cache_disk_blank_tile_key(mapcache_context *ctx, mapcache_tile *tile, unsigned char *color, char **path) +static void _mapcache_cache_disk_blank_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, unsigned char *color, char **path) { /* not implemented for template caches, as symlink_blank will never be set */ *path = apr_psprintf(ctx->pool,"%s/%s/%s/blanks/%02X%02X%02X%02X.%s", - ((mapcache_cache_disk*)tile->tileset->cache)->base_directory, + cache->base_directory, tile->tileset->name, tile->grid_link->grid->name, color[0], @@ -154,12 +154,11 @@ * \param r * \private \memberof mapcache_cache_disk */ -static void _mapcache_cache_disk_tilecache_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_tilecache_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { - mapcache_cache_disk *dcache = (mapcache_cache_disk*)tile->tileset->cache; - if(dcache->base_directory) { + if(cache->base_directory) { char *start; - _mapcache_cache_disk_base_tile_key(ctx, tile, &start); + _mapcache_cache_disk_base_tile_key(ctx, cache, tile, &start); *path = apr_psprintf(ctx->pool,"%s/%02d/%03d/%03d/%03d/%03d/%03d/%03d.%s", start, tile->z, @@ -171,7 +170,7 @@ tile->y % 1000, tile->tileset->format?tile->tileset->format->extension:"png"); } else { - *path = dcache->filename_template; + *path = cache->filename_template; *path = mapcache_util_str_replace(ctx->pool,*path, "{tileset}", tile->tileset->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{ext}", @@ -222,11 +221,10 @@ } } -static void _mapcache_cache_disk_template_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_template_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { - mapcache_cache_disk *dcache = (mapcache_cache_disk*)tile->tileset->cache; - *path = dcache->filename_template; + *path = cache->filename_template; *path = mapcache_util_str_replace(ctx->pool,*path, "{tileset}", tile->tileset->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{ext}", @@ -278,12 +276,11 @@ } } -static void _mapcache_cache_disk_arcgis_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_arcgis_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { - mapcache_cache_disk *dcache = (mapcache_cache_disk*)tile->tileset->cache; - if(dcache->base_directory) { + if(cache->base_directory) { char *start; - _mapcache_cache_disk_base_tile_key(ctx, tile, &start); + _mapcache_cache_disk_base_tile_key(ctx, cache, tile, &start); *path = apr_psprintf(ctx->pool,"%s/L%02d/R%08x/C%08x.%s" , start, tile->z, @@ -297,13 +294,31 @@ } } +static void _mapcache_cache_disk_worldwind_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) +{ + if(cache->base_directory) { + *path = apr_psprintf(ctx->pool,"%s/%d/%04d/%04d_%04d.%s" , + cache->base_directory, + tile->z, + tile->y, + tile->y, + tile->x, + tile->tileset->format?tile->tileset->format->extension:"png"); + } + + if(!*path) { + ctx->set_error(ctx,500, "failed to allocate tile key"); + } +} + -static int _mapcache_cache_disk_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_disk_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; apr_finfo_t finfo; int rv; - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; + cache->tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } @@ -315,12 +330,13 @@ } } -static void _mapcache_cache_disk_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_disk_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { apr_status_t ret; char errmsg[120]; char *filename; - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; + cache->tile_key(ctx, cache, tile, &filename); GC_CHECK_ERROR(ctx); ret = apr_file_remove(filename,ctx->pool); @@ -337,7 +353,7 @@ * \private \memberof mapcache_cache_disk * \sa mapcache_cache::tile_get() */ -static int _mapcache_cache_disk_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_disk_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; apr_file_t *f; @@ -345,11 +361,13 @@ apr_status_t rv; apr_size_t size; apr_mmap_t *tilemmap; + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + cache->tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FAILURE; } + ctx->log(ctx,MAPCACHE_DEBUG,"checking for tile %s",filename); if((rv=apr_file_open(&f, filename, #ifndef NOMMAP APR_FOPEN_READ, APR_UREAD | APR_GREAD, @@ -418,14 +436,15 @@ * \private \memberof mapcache_cache_disk * \sa mapcache_cache::tile_set() */ -static void _mapcache_cache_disk_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_disk_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { apr_size_t bytes; apr_file_t *f; apr_status_t ret; char errmsg[120]; - char *filename, *hackptr1, *hackptr2=NULL; - const int creation_retry = ((mapcache_cache_disk*)tile->tileset->cache)->creation_retry; + char *filename; + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; + const int creation_retry = cache->creation_retry; int retry_count_create_file = 0; #ifdef DEBUG @@ -440,29 +459,11 @@ } #endif - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + cache->tile_key(ctx, cache, tile, &filename); GC_CHECK_ERROR(ctx); - /* find the location of the last '/' in the string */ - hackptr1 = filename; - while(*hackptr1) { - if(*hackptr1 == '/') - hackptr2 = hackptr1; - hackptr1++; - } - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - /* - * apr_dir_make_recursive sometimes sends back this error, although it should not. - * ignore this one - */ - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; - } - } - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); ret = apr_file_remove(filename,ctx->pool); if(ret != APR_SUCCESS && !APR_STATUS_IS_ENOENT(ret)) { @@ -471,34 +472,29 @@ #ifdef HAVE_SYMLINK - if(((mapcache_cache_disk*)tile->tileset->cache)->symlink_blank) { + if(cache->symlink_blank) { if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); } if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { char *blankname; - _mapcache_cache_disk_blank_tile_key(ctx,tile,tile->raw_image->data,&blankname); + int retry_count_create_symlink = 0; + char *blankname_rel = NULL; + _mapcache_cache_disk_blank_tile_key(ctx,cache,tile,tile->raw_image->data,&blankname); if(apr_file_open(&f, blankname, APR_FOPEN_READ, APR_OS_DEFAULT, ctx->pool) != APR_SUCCESS) { + /* create the blank file */ + int isLocked; + void *lock; if(!tile->encoded_data) { tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); GC_CHECK_ERROR(ctx); } - /* create the blank file */ - char *blankdirname = apr_psprintf(ctx->pool, "%s/%s/%s/blanks", - ((mapcache_cache_disk*)tile->tileset->cache)->base_directory, - tile->tileset->name, - tile->grid_link->grid->name); - if(APR_SUCCESS != (ret = apr_dir_make_recursive( - blankdirname, APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create directory %s for blank tiles",blankdirname, apr_strerror(ret,errmsg,120)); - return; - } - } + mapcache_make_parent_dirs(ctx,blankname); + GC_CHECK_ERROR(ctx); /* aquire a lock on the blank file */ - int isLocked = mapcache_lock_or_wait_for_resource(ctx,blankname); + isLocked = mapcache_lock_or_wait_for_resource(ctx,ctx->config->locker,blankname, &lock); if(isLocked == MAPCACHE_TRUE) { @@ -506,7 +502,7 @@ APR_FOPEN_CREATE|APR_FOPEN_WRITE|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY, APR_OS_DEFAULT, ctx->pool)) != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to create file %s: %s",blankname, apr_strerror(ret,errmsg,120)); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); return; /* we could not create the file */ } @@ -514,17 +510,17 @@ ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",blankname, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120)); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); return; /* we could not create the file */ } if(bytes != tile->encoded_data->size) { ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", blankname, (int)bytes, (int)tile->encoded_data->size); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); return; } apr_file_close(f); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); #ifdef DEBUG ctx->log(ctx,MAPCACHE_DEBUG,"created blank tile %s",blankname); #endif @@ -533,12 +529,10 @@ apr_file_close(f); } - int retry_count_create_symlink = 0; /* * compute the relative path between tile and blank tile */ - char *blankname_rel = NULL; blankname_rel = relative_path(ctx,filename, blankname); GC_CHECK_ERROR(ctx); @@ -555,17 +549,8 @@ ctx->set_error(ctx, 500, "failed to link tile %s to %s: %s",filename, blankname_rel, error); return; /* we could not create the file */ } - - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create symlink, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ - } - } - - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); } #ifdef DEBUG ctx->log(ctx, MAPCACHE_DEBUG, "linked blank tile %s to %s",filename,blankname); @@ -597,17 +582,8 @@ ctx->set_error(ctx, 500, "failed to create file %s: %s",filename, apr_strerror(ret,errmsg,120)); return; /* we could not create the file */ } - - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create file, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ - } - } - - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); } bytes = (apr_size_t)tile->encoded_data->size; @@ -643,6 +619,8 @@ dcache->tile_key = _mapcache_cache_disk_tilecache_tile_key; } else if(!strcmp(layout,"arcgis")) { dcache->tile_key = _mapcache_cache_disk_arcgis_tile_key; + } else if(!strcmp(layout,"worldwind")) { + dcache->tile_key = _mapcache_cache_disk_worldwind_tile_key; } else if (!strcmp(layout,"template")) { dcache->tile_key = _mapcache_cache_disk_template_tile_key; template_layout = MAPCACHE_TRUE; diff -Nru mapcache-1.2.1/lib/cache_fallback.c mapcache-1.4.1/lib/cache_fallback.c --- mapcache-1.2.1/lib/cache_fallback.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/cache_fallback.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,202 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: fallback cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without fallbackriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" + +static int _mapcache_cache_fallback_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); + return subcache->tile_exists(ctx, subcache, tile); +} + +static void _mapcache_cache_fallback_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + int i; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + subcache->tile_delete(ctx, subcache, tile); + ctx->clear_errors(ctx); /* ignore errors */ + } +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the fallback server + * \private \memberof mapcache_cache_fallback + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_fallback_tile_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + mapcache_cache *subcache; + int i,ret; + subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); + ret = subcache->tile_get(ctx, subcache, tile); + + if(ret == MAPCACHE_FAILURE) { + int first_error = ctx->get_error(ctx); + char *first_error_message = ctx->get_error_message(ctx); + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"GET\" on primary cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Falling back on secondary caches", + APR_ARRAY_IDX(cache->caches,0,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); + ctx->clear_errors(ctx); + for(i=1; icaches->nelts; i++) { + subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if((ret = subcache->tile_get(ctx, subcache, tile)) == MAPCACHE_FAILURE) { + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"GET\" on fallback cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Continuing with other fallback caches if available", + APR_ARRAY_IDX(cache->caches,0,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); + ctx->clear_errors(ctx); + continue; + } else { + return ret; + } + } + /* all backends failed, return primary error message */ + ctx->set_error(ctx,first_error,first_error_message); + return MAPCACHE_FAILURE; + } else { + /* success or notfound */ + return ret; + } +} + +static void _mapcache_cache_fallback_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + int i,oneok=0,first_error=0; + char *first_error_message; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + subcache->tile_set(ctx, subcache, tile); + if(GC_HAS_ERROR(ctx)) { + if(!first_error) { + first_error = ctx->get_error(ctx); + first_error_message = ctx->get_error_message(ctx); + } + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"SET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", + APR_ARRAY_IDX(cache->caches,i,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); + ctx->clear_errors(ctx); + } else { + oneok = 1; + } + } + if(!oneok) { + ctx->set_error(ctx,first_error,first_error_message); + } +} + +static void _mapcache_cache_fallback_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + int i,oneok=0,first_error=0; + char *first_error_message; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if(subcache->tile_multi_set) { + subcache->tile_multi_set(ctx, subcache, tiles, ntiles); + } + else { + int j; + for(j=0;jtile_set(ctx,subcache,&tiles[j]); + if(GC_HAS_ERROR(ctx)) + break; + } + } + if(GC_HAS_ERROR(ctx)) { + if(!first_error) { + first_error = ctx->get_error(ctx); + first_error_message = ctx->get_error_message(ctx); + } + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"MULTISET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", + APR_ARRAY_IDX(cache->caches,i,mapcache_cache*)->name,tiles[0].z,tiles[0].x,tiles[0].y,tiles[0].tileset->name); + ctx->clear_errors(ctx); + } else { + oneok = 1; + } + } + if(!oneok) { + ctx->set_error(ctx,first_error,first_error_message); + } +} + +/** + * \private \memberof mapcache_cache_fallback + */ +static void _mapcache_cache_fallback_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + cache->caches = apr_array_make(ctx->pool,3,sizeof(mapcache_cache*)); + for(cur_node = ezxml_child(node,"cache"); cur_node; cur_node = cur_node->next) { + mapcache_cache *refcache = mapcache_configuration_get_cache(config, cur_node->txt); + if(!refcache) { + ctx->set_error(ctx, 400, "fallback cache \"%s\" references cache \"%s\"," + " but it is not configured (hint:referenced caches must be declared before this fallback cache in the xml file)", pcache->name, cur_node->txt); + return; + } + APR_ARRAY_PUSH(cache->caches,mapcache_cache*) = refcache; + } + if(cache->caches->nelts == 0) { + ctx->set_error(ctx,400,"fallback cache \"%s\" does not reference any child caches", pcache->name); + } +} + +/** + * \private \memberof mapcache_cache_fallback + */ +static void _mapcache_cache_fallback_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ +} + + +/** + * \brief creates and initializes a mapcache_cache_fallback + */ +mapcache_cache* mapcache_cache_fallback_create(mapcache_context *ctx) +{ + mapcache_cache_fallback *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_fallback)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate fallback cache"); + return NULL; + } + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_COMPOSITE; + cache->cache.tile_delete = _mapcache_cache_fallback_tile_delete; + cache->cache.tile_get = _mapcache_cache_fallback_tile_get; + cache->cache.tile_exists = _mapcache_cache_fallback_tile_exists; + cache->cache.tile_set = _mapcache_cache_fallback_tile_set; + cache->cache.tile_multi_set = _mapcache_cache_fallback_tile_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_fallback_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_fallback_configuration_parse_xml; + return (mapcache_cache*)cache; +} + diff -Nru mapcache-1.2.1/lib/cache_memcache.c mapcache-1.4.1/lib/cache_memcache.c --- mapcache-1.2.1/lib/cache_memcache.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/cache_memcache.c 2016-02-25 15:47:16.000000000 +0000 @@ -31,41 +31,117 @@ #ifdef USE_MEMCACHE #include "mapcache.h" +struct mapcache_memcache_conn_param { + mapcache_cache_memcache *cache; +}; + +struct mapcache_memcache_pooled_connection { + apr_memcache_t *memcache; + apr_pool_t *pool; +}; + +void mapcache_memcache_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + struct mapcache_memcache_conn_param *param = params; + mapcache_cache_memcache *cache = param->cache; + struct mapcache_memcache_pooled_connection *pc; + int i; + pc = calloc(1,sizeof(struct mapcache_memcache_pooled_connection)); + apr_pool_create(&pc->pool,NULL); + if(APR_SUCCESS != apr_memcache_create(pc->pool, cache->nservers, 0, &(pc->memcache))) { + ctx->set_error(ctx,500,"cache %s: failed to create memcache backend", cache->cache.name); + return; + } + for(i=0; icache->nservers; i++) { + apr_memcache_server_t *server; + if(APR_SUCCESS != apr_memcache_server_create(pc->pool,cache->servers[i].host,cache->servers[i].port,4,5,50,10000,&server)) { + ctx->set_error(ctx,500,"cache %s: failed to create server %s:%d",cache->cache.name,cache->servers[i].host,cache->servers[i].port); + return; + } + if(APR_SUCCESS != apr_memcache_add_server(pc->memcache,server)) { + ctx->set_error(ctx,500,"cache %s: failed to add server %s:%d",cache->cache.name,cache->servers[i].host,cache->servers[i].port); + return; + } + } + *conn_ = pc; +} + +void mapcache_memcache_connection_destructor(void *conn_) { + struct mapcache_memcache_pooled_connection *pc = conn_; + apr_pool_destroy(pc->pool); + free(pc); +} + +static mapcache_pooled_connection* _mapcache_memcache_get_conn(mapcache_context *ctx, + mapcache_cache_memcache *cache, mapcache_tile *tile) { + mapcache_pooled_connection *pc; + struct mapcache_memcache_conn_param param; + + param.cache = cache; -static int _mapcache_cache_memcache_has_tile(mapcache_context *ctx, mapcache_tile *tile) + pc = mapcache_connection_pool_get_connection(ctx,cache->cache.name, mapcache_memcache_connection_constructor, mapcache_memcache_connection_destructor, ¶m); + return pc; +} + +static void _mapcache_memcache_release_conn(mapcache_context *ctx, mapcache_pooled_connection *con) { + mapcache_connection_pool_release_connection(ctx, con); +} + +static int _mapcache_cache_memcache_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *key; char *tmpdata; int rv; size_t tmpdatasize; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); - if(GC_HAS_ERROR(ctx)) { + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); + if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; + mpc = pc->connection; + + key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) { + rv = MAPCACHE_FALSE; + goto cleanup; } - rv = apr_memcache_getp(cache->memcache,ctx->pool,key,&tmpdata,&tmpdatasize,NULL); + rv = apr_memcache_getp(mpc->memcache,ctx->pool,key,&tmpdata,&tmpdatasize,NULL); if(rv != APR_SUCCESS) { - return MAPCACHE_FALSE; + rv = MAPCACHE_FALSE; + goto cleanup; } if(tmpdatasize == 0) { - return MAPCACHE_FALSE; + rv = MAPCACHE_FALSE; + goto cleanup; } - return MAPCACHE_TRUE; + rv = MAPCACHE_TRUE; +cleanup: + _mapcache_memcache_release_conn(ctx,pc); + return rv; } -static void _mapcache_cache_memcache_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_memcache_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *key; int rv; char errmsg[120]; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); GC_CHECK_ERROR(ctx); - rv = apr_memcache_delete(cache->memcache,key,0); + mpc = pc->connection; + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) goto cleanup; + + rv = apr_memcache_delete(mpc->memcache,key,0); if(rv != APR_SUCCESS && rv!= APR_NOTFOUND) { - int code = 500; - ctx->set_error(ctx,code,"memcache: failed to delete key %s: %s", key, apr_strerror(rv,errmsg,120)); + ctx->set_error(ctx,500,"memcache: failed to delete key %s: %s", key, apr_strerror(rv,errmsg,120)); + goto cleanup; } + +cleanup: + _mapcache_memcache_release_conn(ctx,pc); } /** @@ -75,33 +151,55 @@ * \private \memberof mapcache_cache_memcache * \sa mapcache_cache::tile_get() */ -static int _mapcache_cache_memcache_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_memcache_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *key; int rv; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + mapcache_buffer *encoded_data; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FAILURE; } - tile->encoded_data = mapcache_buffer_create(0,ctx->pool); - rv = apr_memcache_getp(cache->memcache,ctx->pool,key,(char**)&tile->encoded_data->buf,&tile->encoded_data->size,NULL); + mpc = pc->connection; + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) { + rv = MAPCACHE_FAILURE; + goto cleanup; + } + encoded_data = mapcache_buffer_create(0,ctx->pool); + rv = apr_memcache_getp(mpc->memcache,ctx->pool,key,(char**)&encoded_data->buf,&encoded_data->size,NULL); if(rv != APR_SUCCESS) { - return MAPCACHE_CACHE_MISS; + rv = MAPCACHE_CACHE_MISS; + goto cleanup; } - if(tile->encoded_data->size == 0) { + if(encoded_data->size == 0) { ctx->set_error(ctx,500,"memcache cache returned 0-length data for tile %d %d %d\n",tile->x,tile->y,tile->z); - return MAPCACHE_FAILURE; + rv = MAPCACHE_FAILURE; + goto cleanup; } /* extract the tile modification time from the end of the data returned */ memcpy( &tile->mtime, - &(((char*)tile->encoded_data->buf)[tile->encoded_data->size-sizeof(apr_time_t)]), + &(((char*)encoded_data->buf)[encoded_data->size-sizeof(apr_time_t)]), sizeof(apr_time_t)); - ((char*)tile->encoded_data->buf)[tile->encoded_data->size+sizeof(apr_time_t)]='\0'; - tile->encoded_data->avail = tile->encoded_data->size; - tile->encoded_data->size -= sizeof(apr_time_t); - return MAPCACHE_SUCCESS; + + ((char*)encoded_data->buf)[encoded_data->size-sizeof(apr_time_t)]='\0'; + encoded_data->avail = encoded_data->size; + encoded_data->size -= sizeof(apr_time_t); + if(((char*)encoded_data->buf)[0] == '#' && encoded_data->size > 1) { + tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy ,encoded_data->buf,&tile->nodata); + } else { + tile->encoded_data = encoded_data; + } + rv = MAPCACHE_SUCCESS; + +cleanup: + _mapcache_memcache_release_conn(ctx,pc); + + return rv; } /** @@ -113,37 +211,63 @@ * \private \memberof mapcache_cache_memcache * \sa mapcache_cache::tile_set() */ -static void _mapcache_cache_memcache_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_memcache_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - char *key; + char *key, *data; int rv; - /* set expiration to one day if not configured */ - int expires = 86400; + /* set no expiration if not configured */ + int expires =0; + apr_time_t now; + mapcache_buffer *encoded_data = NULL; + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); + GC_CHECK_ERROR(ctx); + mpc = pc->connection; + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) goto cleanup; + if(tile->tileset->auto_expire) expires = tile->tileset->auto_expire; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); - GC_CHECK_ERROR(ctx); - if(!tile->encoded_data) { - tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); - GC_CHECK_ERROR(ctx); + if(cache->detect_blank) { + if(!tile->raw_image) { + tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); + GC_CHECK_ERROR(ctx); + } + if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { + encoded_data = mapcache_buffer_create(5,ctx->pool); + ((char*)encoded_data->buf)[0] = '#'; + memcpy(((char*)encoded_data->buf)+1,tile->raw_image->data,4); + encoded_data->size = 5; + } + } + if(!encoded_data) { + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + if(GC_HAS_ERROR(ctx)) goto cleanup; + } + encoded_data = tile->encoded_data; } /* concatenate the current time to the end of the memcache data so we can extract it out * when we re-get the tile */ - char *data = calloc(1,tile->encoded_data->size+sizeof(apr_time_t)); - apr_time_t now = apr_time_now(); + data = calloc(1,encoded_data->size+sizeof(apr_time_t)); + now = apr_time_now(); apr_pool_cleanup_register(ctx->pool, data, (void*)free, apr_pool_cleanup_null); - memcpy(data,tile->encoded_data->buf,tile->encoded_data->size); - memcpy(&(data[tile->encoded_data->size]),&now,sizeof(apr_time_t)); + memcpy(data,encoded_data->buf,encoded_data->size); + memcpy(&(data[encoded_data->size]),&now,sizeof(apr_time_t)); - rv = apr_memcache_set(cache->memcache,key,data,tile->encoded_data->size+sizeof(apr_time_t),expires,0); + rv = apr_memcache_set(mpc->memcache,key,data,encoded_data->size+sizeof(apr_time_t),expires,0); if(rv != APR_SUCCESS) { ctx->set_error(ctx,500,"failed to store tile %d %d %d to memcache cache %s", tile->x,tile->y,tile->z,cache->cache.name); - return; + goto cleanup; } + +cleanup: + _mapcache_memcache_release_conn(ctx,pc); } /** @@ -152,30 +276,25 @@ static void _mapcache_cache_memcache_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { ezxml_t cur_node; + int i = 0; mapcache_cache_memcache *dcache = (mapcache_cache_memcache*)cache; - int servercount = 0; for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { - servercount++; + dcache->nservers++; } - if(!servercount) { + if(!dcache->nservers) { ctx->set_error(ctx,400,"memcache cache %s has no s configured",cache->name); return; } - if(APR_SUCCESS != apr_memcache_create(ctx->pool, servercount, 0, &dcache->memcache)) { - ctx->set_error(ctx,400,"cache %s: failed to create memcache backend", cache->name); - return; - } + dcache->servers = apr_pcalloc(ctx->pool, dcache->nservers * sizeof(struct mapcache_cache_memcache_server)); + for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { ezxml_t xhost = ezxml_child(cur_node,"host"); ezxml_t xport = ezxml_child(cur_node,"port"); - const char *host; - apr_memcache_server_t *server; - apr_port_t port; if(!xhost || !xhost->txt || ! *xhost->txt) { ctx->set_error(ctx,400,"cache %s: with no ",cache->name); return; } else { - host = apr_pstrdup(ctx->pool,xhost->txt); + dcache->servers[i].host = apr_pstrdup(ctx->pool,xhost->txt); } if(!xport || !xport->txt || ! *xport->txt) { @@ -188,19 +307,15 @@ ctx->set_error(ctx,400,"failed to parse value %s for memcache cache %s", xport->txt,cache->name); return; } - port = iport; + dcache->servers[i].port = iport; } - if(APR_SUCCESS != apr_memcache_server_create(ctx->pool,host,port,4,5,50,10000,&server)) { - ctx->set_error(ctx,400,"cache %s: failed to create server %s:%d",cache->name,host,port); - return; - } - if(APR_SUCCESS != apr_memcache_add_server(dcache->memcache,server)) { - ctx->set_error(ctx,400,"cache %s: failed to add server %s:%d",cache->name,host,port); - return; - } - if(APR_SUCCESS != apr_memcache_set(dcache->memcache,"mapcache_test_key","mapcache",8,0,0)) { - ctx->set_error(ctx,400,"cache %s: failed to add test key to server %s:%d",cache->name,host,port); - return; + i++; + } + + dcache->detect_blank = 0; + if ((cur_node = ezxml_child(node, "detect_blank")) != NULL) { + if(!strcasecmp(cur_node->txt,"true")) { + dcache->detect_blank = 1; } } } @@ -212,7 +327,7 @@ mapcache_cfg *cfg) { mapcache_cache_memcache *dcache = (mapcache_cache_memcache*)cache; - if(!dcache->memcache || dcache->memcache->ntotal==0) { + if(!dcache->nservers) { ctx->set_error(ctx,400,"cache %s has no servers configured",cache->name); } } diff -Nru mapcache-1.2.1/lib/cache_multitier.c mapcache-1.4.1/lib/cache_multitier.c --- mapcache-1.2.1/lib/cache_multitier.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/cache_multitier.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,165 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: multitier cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without multitierriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" + +static int _mapcache_cache_multitier_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + int i; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if(subcache->tile_exists(ctx, subcache, tile) == MAPCACHE_TRUE) { + return MAPCACHE_TRUE; + } + } + return MAPCACHE_FALSE; +} + +static void _mapcache_cache_multitier_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + int i; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + subcache->tile_delete(ctx, subcache, tile); + ctx->clear_errors(ctx); /* ignore errors */ + } +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the multitier server + * \private \memberof mapcache_cache_multitier + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_multitier_tile_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + mapcache_cache *subcache; + int i,ret; + subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); + ret = subcache->tile_get(ctx, subcache, tile); + + if(ret == MAPCACHE_CACHE_MISS) { + for(i=1; icaches->nelts; i++) { + subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if(subcache->tile_get(ctx, subcache, tile) == MAPCACHE_SUCCESS) { + ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from secondary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + for(--i;i>=0;i--) { + subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + subcache->tile_set(ctx, subcache, tile); + ctx->clear_errors(ctx); /* silently ignore these errors */ + ctx->log(ctx,MAPCACHE_DEBUG,"transferring tile (%s,z=%d,y=%d,x=%d) to cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + } + return MAPCACHE_SUCCESS; + } + } + return MAPCACHE_CACHE_MISS; + } else { + //ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from primary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + return ret; + } +} + +static void _mapcache_cache_multitier_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,cache->caches->nelts-1,mapcache_cache*); + return subcache->tile_set(ctx, subcache, tile); +} + +static void _mapcache_cache_multitier_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,cache->caches->nelts-1,mapcache_cache*); + if(subcache->tile_multi_set) { + return subcache->tile_multi_set(ctx, subcache, tiles, ntiles); + } else { + int i; + for( i=0;itile_set(ctx, subcache, &tiles[i]); + GC_CHECK_ERROR(ctx); + } + } +} + +/** + * \private \memberof mapcache_cache_multitier + */ +static void _mapcache_cache_multitier_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + cache->caches = apr_array_make(ctx->pool,3,sizeof(mapcache_cache*)); + for(cur_node = ezxml_child(node,"cache"); cur_node; cur_node = cur_node->next) { + mapcache_cache *refcache = mapcache_configuration_get_cache(config, cur_node->txt); + if(!refcache) { + ctx->set_error(ctx, 400, "multitier cache \"%s\" references cache \"%s\"," + " but it is not configured (hint:referenced caches must be declared before this multitier cache in the xml file)", pcache->name, cur_node->txt); + return; + } + APR_ARRAY_PUSH(cache->caches,mapcache_cache*) = refcache; + } + if(cache->caches->nelts == 0) { + ctx->set_error(ctx,400,"multitier cache \"%s\" does not reference any child caches", pcache->name); + } +} + +/** + * \private \memberof mapcache_cache_multitier + */ +static void _mapcache_cache_multitier_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ +} + + +/** + * \brief creates and initializes a mapcache_cache_multitier + */ +mapcache_cache* mapcache_cache_multitier_create(mapcache_context *ctx) +{ + mapcache_cache_multitier *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_multitier)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate multitier cache"); + return NULL; + } + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_COMPOSITE; + cache->cache.tile_delete = _mapcache_cache_multitier_tile_delete; + cache->cache.tile_get = _mapcache_cache_multitier_tile_get; + cache->cache.tile_exists = _mapcache_cache_multitier_tile_exists; + cache->cache.tile_set = _mapcache_cache_multitier_tile_set; + cache->cache.tile_multi_set = _mapcache_cache_multitier_tile_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_multitier_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_multitier_configuration_parse_xml; + return (mapcache_cache*)cache; +} + diff -Nru mapcache-1.2.1/lib/cache_rest.c mapcache-1.4.1/lib/cache_rest.c --- mapcache-1.2.1/lib/cache_rest.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/cache_rest.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,1176 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching: HTTP Rest cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 2014 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" +#include +#include +#include +#include +#include +#include +#include + +typedef struct { + mapcache_buffer *buffer; + size_t offset; +} buffer_struct; + +static size_t buffer_read_callback(void *ptr, size_t size, size_t nmemb, void *stream) +{ + buffer_struct *buffer = (buffer_struct*)stream; + void *start = ((char*)(buffer->buffer->buf)) + buffer->offset; + size_t bytes = MAPCACHE_MIN((buffer->buffer->size-buffer->offset),(size * nmemb)); + if(bytes) { + memcpy(ptr,start,bytes); + buffer->offset += bytes; + } + return bytes; +} + +size_t buffer_write_callback(void *ptr, size_t size, size_t nmemb, void *data) +{ + mapcache_buffer *buffer = (mapcache_buffer*)data; + size_t realsize = size * nmemb; + return mapcache_buffer_append(buffer, realsize, ptr); +} + +static void _set_headers(mapcache_context *ctx, CURL *curl, apr_table_t *headers) { + if(!headers) { + return; + } else { + struct curl_slist *curl_headers=NULL; + const apr_array_header_t *array = apr_table_elts(headers); + apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; + int i; + for (i = 0; i < array->nelts; i++) { + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",elts[i].val,NULL)); + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); + } +} + +static void _put_request(mapcache_context *ctx, CURL *curl, mapcache_buffer *buffer, char *url, apr_table_t *headers) { + CURLcode res; + buffer_struct data; + mapcache_buffer *response; + + data.buffer = buffer; + data.offset = 0; + + response = mapcache_buffer_create(10,ctx->pool); + +#if LIBCURL_VERSION_NUM < 0x071700 + /* + * hack around a bug in curl <= 7.22 where the content-length is added + * a second time even if ti was present in the manually set headers + */ + apr_table_unset(headers, "Content-Length"); +#endif + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + /* we want to use our own read function */ + curl_easy_setopt(curl, CURLOPT_READFUNCTION, buffer_read_callback); + + /* enable uploading */ + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + + /* HTTP PUT please */ + curl_easy_setopt(curl, CURLOPT_PUT, 1L); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + /* now specify which file to upload */ + curl_easy_setopt(curl, CURLOPT_READDATA, &data); + + /* provide the size of the upload, we specicially typecast the value + * to curl_off_t since we must be sure to use the correct data size */ + curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer->size); + + /* send all data to this function */ + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, buffer_write_callback); + + /* we pass our mapcache_buffer struct to the callback function */ + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)response); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest put: %s",curl_easy_strerror(res)); + } else { + long http_code; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if(http_code != 200 && http_code != 201 && http_code != 204) { + char *msg = response->buf; + msg[response->size]=0; + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest put with code %ld: %s", http_code, msg); + } + } + +} + +static int _head_request(mapcache_context *ctx, char *url, apr_table_t *headers) { + + CURL *curl; + CURLcode res; + long http_code; + + curl = curl_easy_init(); + + if(!curl) { + ctx->set_error(ctx,500,"failed to create curl handle"); + return -1; + } + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest head %s",curl_easy_strerror(res)); + http_code = 500; + } else { + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + } + + /* always cleanup */ + curl_easy_cleanup(curl); + + return (int)http_code; +} + +static int _delete_request(mapcache_context *ctx, char *url, apr_table_t *headers) { + + CURL *curl; + CURLcode res; + long http_code; + + curl = curl_easy_init(); + if(!curl) { + ctx->set_error(ctx,500,"failed to create curl handle"); + return -1; + } + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest head %s",curl_easy_strerror(res)); + http_code = 500; + } else { + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + } + + /* always cleanup */ + curl_easy_cleanup(curl); + + return (int)http_code; +} + +static mapcache_buffer* _get_request(mapcache_context *ctx, char *url, apr_table_t *headers) { + + CURL *curl; + CURLcode res; + mapcache_buffer *data = NULL; + long http_code; + + curl = curl_easy_init(); + if(!curl) { + ctx->set_error(ctx,500,"failed to create curl handle"); + return NULL; + } + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + data = mapcache_buffer_create(4000, ctx->pool); + + /* send all data to this function */ + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, buffer_write_callback); + + /* we pass our mapcache_buffer struct to the callback function */ + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)data); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get: %s",curl_easy_strerror(res)); + data = NULL; + } else { + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + /* handle special behavior of s3 */ + if(http_code == 403) { + char *msg = data->buf; + while(msg && *msg) { + if(!strncmp(msg,"NoSuchKey",strlen("NoSuchKey"))) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get with code %ld: %s", http_code, msg); + http_code = 404; + data = NULL; + break; + } + msg++; + } + } + if(http_code != 200 && http_code != 404) { + char *msg = data->buf; + msg[data->size]=0; + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get with code %ld: %s", http_code, msg); + } + if(http_code == 404) { + data = NULL; /* not an error */ + } + } + + /* always cleanup */ + curl_easy_cleanup(curl); + + return data; +} + +apr_table_t* _mapcache_cache_rest_headers(mapcache_context *ctx, mapcache_tile *tile, mapcache_rest_configuration *config, + mapcache_rest_operation *operation) { + apr_table_t *ret = apr_table_make(ctx->pool,3); + const apr_array_header_t *array; + + if(config->common_headers) { + apr_table_entry_t *elts; + int i; + array = apr_table_elts(config->common_headers); + elts = (apr_table_entry_t *) array->elts; + for (i = 0; i < array->nelts; i++) { + apr_table_set(ret, elts[i].key,elts[i].val); + } + } + if(operation->headers) { + apr_table_entry_t *elts; + int i; + array = apr_table_elts(operation->headers); + elts = (apr_table_entry_t *) array->elts; + for (i = 0; i < array->nelts; i++) { + apr_table_set(ret, elts[i].key,elts[i].val); + } + } + return ret; +} + +/* Converts an integer value to its hex character*/ +static char to_hex(char code) { + static char hex[] = "0123456789ABCDEF"; + return hex[code & 15]; +} + +/* Returns a url-encoded version of str */ +static char *url_encode(apr_pool_t *pool, char *str) { + char *pstr = str, *buf = apr_pcalloc(pool, strlen(str) * 3 + 1), *pbuf = buf; + while (*pstr) { + if (isalnum(*pstr) || *pstr == '-' || *pstr == '_' || *pstr == '.' || *pstr == '~' || *pstr=='/') + *pbuf++ = *pstr; + else if (*pstr == ' ') + *pbuf++ = '+'; + else + *pbuf++ = '%', *pbuf++ = to_hex(*pstr >> 4), *pbuf++ = to_hex(*pstr & 15); + pstr++; + } + *pbuf = '\0'; + return buf; +} +/** + * \brief return url for given tile given a template + * + * \param tile the tile to get the key from + * \param template the template to build the url from + * \param path pointer to a char* that will contain the url + * \param r + * \private \memberof mapcache_cache_rest + */ +static void _mapcache_cache_rest_tile_url(mapcache_context *ctx, mapcache_tile *tile, mapcache_rest_configuration *config, + mapcache_rest_operation *operation, char **url) +{ + char *slashptr,*path; + int cnt=0; + if(operation && operation->tile_url) { + *url = apr_pstrdup(ctx->pool, operation->tile_url); + } else { + *url = apr_pstrdup(ctx->pool, config->tile_url); + } + + *url = mapcache_util_str_replace(ctx->pool, *url, "{tileset}", tile->tileset->name); + *url = mapcache_util_str_replace(ctx->pool, *url, "{grid}", tile->grid_link->grid->name); + *url = mapcache_util_str_replace(ctx->pool, *url, "{ext}", + tile->tileset->format?tile->tileset->format->extension:"png"); + if(strstr(*url,"{x}")) + *url = mapcache_util_str_replace(ctx->pool,*url, "{x}", + apr_psprintf(ctx->pool,"%d",tile->x)); + else + *url = mapcache_util_str_replace(ctx->pool,*url, "{inv_x}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)); + if(strstr(*url,"{y}")) + *url = mapcache_util_str_replace(ctx->pool,*url, "{y}", + apr_psprintf(ctx->pool,"%d",tile->y)); + else + *url = mapcache_util_str_replace(ctx->pool,*url, "{inv_y}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)); + if(strstr(*url,"{z}")) + *url = mapcache_util_str_replace(ctx->pool,*url, "{z}", + apr_psprintf(ctx->pool,"%d",tile->z)); + else + *url = mapcache_util_str_replace(ctx->pool,*url, "{inv_z}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->nlevels - tile->z - 1)); + if(tile->dimensions) { + char *dimstring=""; + const apr_array_header_t *elts = apr_table_elts(tile->dimensions); + int i = elts->nelts; + while(i--) { + apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->key,"#",entry->val,NULL); + } + *url = mapcache_util_str_replace(ctx->pool,*url, "{dim}", dimstring); + } + /* url-encode everything after the host name */ + + /* find occurence of third "/" in url */ + slashptr = *url; + while(*slashptr) { + if(*slashptr == '/') cnt++; + if(cnt == 3) break; + slashptr++; + } + if(!*slashptr) { + ctx->set_error(ctx,500,"invalid rest url provided, expecting http(s)://server/path format"); + return; + } + path=slashptr; + path = url_encode(ctx->pool,path); + *slashptr=0; + + + *url = apr_pstrcat(ctx->pool,*url,path,NULL); + /*ctx->log(ctx,MAPCACHE_WARN,"rest url: %s",*url);*/ +} + + +// Simple comparison function for comparing two HTTP header names that are +// embedded within an HTTP header line, returning true if header1 comes +// before header2 alphabetically, false if not +static int headerle(const char *header1, const char *header2) +{ + while (1) { + if (*header1 == ':') { + return (*header2 == ':'); + } + else if (*header2 == ':') { + return 0; + } + else if (*header2 < *header1) { + return 0; + } + else if (*header2 > *header1) { + return 1; + } + header1++, header2++; + } +} + + +// Replace this with merge sort eventually, it's the best stable sort. But +// since typically the number of elements being sorted is small, it doesn't +// matter that much which sort is used, and gnome sort is the world's simplest +// stable sort. Added a slight twist to the standard gnome_sort - don't go +// forward +1, go forward to the last highest index considered. This saves +// all the string comparisons that would be done "going forward", and thus +// only does the necessary string comparisons to move values back into their +// sorted position. +static void header_gnome_sort(char **headers, int size) +{ + int i = 0, last_highest = 0; + + while (i < size) { + if ((i == 0) || headerle(headers[i - 1], headers[i])) { + i = ++last_highest; + } + else { + char *tmp = headers[i]; + headers[i] = headers[i - 1]; + headers[--i] = tmp; + } + } +} + +static void _mapcache_cache_google_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) +{ + char *stringToSign, **aheaders, *resource = url, x_amz_date[64]; + const char *head; + const apr_array_header_t *ahead; + apr_table_entry_t *elts; + mapcache_cache_google *google; + time_t now; + struct tm *tnow; + unsigned char sha[65]; + char b64[150]; + int i,nCanonicalHeaders=0,cnt=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_GOOGLE); + google = (mapcache_cache_google*)rcache; + now = time(NULL); + tnow = gmtime(&now); + sha[64]=0; + + strftime(x_amz_date, 64 , "%a, %d %b %Y %H:%M:%S GMT", tnow); + apr_table_set(headers,"x-amz-date",x_amz_date); + + if(!strcmp(method,"PUT")) { + assert(tile->encoded_data); + apr_md5(sha,tile->encoded_data->buf,tile->encoded_data->size); + apr_base64_encode(b64, (char*)sha, 16); + apr_table_set(headers, "Content-MD5", b64); + } + + head = apr_table_get(headers, "Content-MD5"); + if(!head) head = ""; + stringToSign=apr_pstrcat(ctx->pool, method, "\n", head, "\n", NULL); + head = apr_table_get(headers, "Content-Type"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + + /* Date: header, left empty as we are using x-amz-date */ + stringToSign=apr_pstrcat(ctx->pool, stringToSign, "\n", NULL); + + ahead = apr_table_elts(headers); + aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); + elts = (apr_table_entry_t *) ahead->elts; + + for (i = 0; i < ahead->nelts; i++) { + if(!strncmp(elts[i].key,"x-amz-",6)) { + char *k = aheaders[nCanonicalHeaders] = apr_pstrdup(ctx->pool, elts[i].key); + while(*k) { + *k = tolower(*k); + k++; + } + nCanonicalHeaders++; + } + } + header_gnome_sort(aheaders, nCanonicalHeaders); + + for(i=0; ipool, stringToSign, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); + } + + /* find occurence of third "/" in url */ + while(*resource) { + if(*resource == '/') cnt++; + if(cnt == 3) break; + resource++; + } + if(!*resource) { + ctx->set_error(ctx,500,"invalid google url provided"); + return; + } + + stringToSign = apr_pstrcat(ctx->pool, stringToSign, resource, NULL); + + hmac_sha1(stringToSign, strlen(stringToSign), (unsigned char*)google->secret, strlen(google->secret), sha); + + apr_base64_encode(b64, (char*)sha, 20); + + + apr_table_set( headers, "Authorization", apr_pstrcat(ctx->pool,"AWS ", google->access, ":", b64, NULL)); +} +static void _mapcache_cache_azure_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) +{ + char *stringToSign, **aheaders, *canonical_headers="", *canonical_resource=NULL, *resource = url, x_ms_date[64]; + const char *head; + const apr_array_header_t *ahead; + apr_table_entry_t *elts; + mapcache_cache_azure *azure; + time_t now; + struct tm *tnow; + unsigned char sha[65]; + char *b64sign,*keyub64; + int i,nCanonicalHeaders=0,cnt=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_AZURE); + azure = (mapcache_cache_azure*)rcache; + now = time(NULL); + tnow = gmtime(&now); + sha[64]=0; + + strftime(x_ms_date, sizeof(x_ms_date), "%a, %d %b %Y %H:%M:%S GMT", tnow); + apr_table_set(headers,"x-ms-date",x_ms_date); + apr_table_set(headers,"x-ms-version","2009-09-19"); + apr_table_set(headers,"x-ms-blob-type","BlockBlob"); + + stringToSign = apr_pstrcat(ctx->pool, method, "\n", NULL); + head = apr_table_get(headers, "Content-Encoding"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "Content-Language"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "Content-Length"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "Content-MD5"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "Content-Type"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "Date"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "If-Modified-Since"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "If-Match"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "If-None-Match"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "If-Unmodified-Since"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = apr_table_get(headers, "Range"); + if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + + ahead = apr_table_elts(headers); + aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); + elts = (apr_table_entry_t *) ahead->elts; + + for (i = 0; i < ahead->nelts; i++) { + char *k; + if(strncmp(elts[i].key,"x-ms-",5) || elts[i].key[5]==0) continue; + k = aheaders[nCanonicalHeaders] = apr_pstrdup(ctx->pool, elts[i].key); + while(*k) { + *k = tolower(*k); + k++; + } + nCanonicalHeaders++; + } + header_gnome_sort(aheaders, nCanonicalHeaders); + + for(i=0; ipool, canonical_headers, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); + } + + /* find occurence of third "/" in url */ + while(*resource) { + if(*resource == '/') cnt++; + if(cnt == 3) break; + resource++; + } + if(!*resource) { + ctx->set_error(ctx,500,"invalid azure url provided"); + return; + } + + canonical_resource = apr_pstrcat(ctx->pool, "/", azure->id, resource, NULL); + + stringToSign = apr_pstrcat(ctx->pool, stringToSign, canonical_headers, canonical_resource, NULL); + + keyub64 = (char*)apr_pcalloc(ctx->pool, apr_base64_decode_len(azure->secret)); + apr_base64_decode(keyub64, azure->secret); + + hmac_sha256((unsigned char*)stringToSign, strlen(stringToSign), (unsigned char*)keyub64, strlen(keyub64), sha, 32); + + b64sign = (char*)apr_pcalloc(ctx->pool, apr_base64_encode_len(32)); + apr_base64_encode(b64sign, (char*)sha, 32); + + + apr_table_set( headers, "Authorization", apr_pstrcat(ctx->pool,"SharedKey ", azure->id, ":", b64sign, NULL)); + + +} +static void _mapcache_cache_s3_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) +{ + unsigned char sha1[65],sha2[65]; + int cnt=0,i; + time_t now = time(NULL); + struct tm *tnow = gmtime(&now); + const apr_array_header_t *ahead; + char *tosign, *key, *canonical_request, x_amz_date[64], *resource = url, **aheaders, *auth; + apr_table_entry_t *elts; + mapcache_cache_s3 *s3; + + sha1[64]=sha2[64]=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_S3); + s3 = (mapcache_cache_s3*)rcache; + + if(!strcmp(method,"PUT")) { + assert(tile->encoded_data); + sha256((unsigned char*)tile->encoded_data->buf, tile->encoded_data->size, sha1); + sha_hex_encode(sha1,32); + } else { + /* sha256 hash of empty string */ + memcpy(sha1,"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",64); + } + apr_table_set(headers,"x-amz-content-sha256", (char*)sha1); + /* sha1 contains the hash of the payload */ + + /* find occurence of third "/" in url */ + while(*resource) { + if(*resource == '/') cnt++; + if(cnt == 3) break; + resource++; + } + if(!*resource) { + ctx->set_error(ctx,500,"invalid s3 url provided"); + return; + } + + strftime(x_amz_date, sizeof(x_amz_date), "%Y%m%dT%H%M%SZ", tnow); + apr_table_set(headers, "x-amz-date", x_amz_date); + + canonical_request = apr_pstrcat(ctx->pool, method, "\n" ,resource, "\n\n",NULL); + + ahead = apr_table_elts(headers); + aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); + elts = (apr_table_entry_t *) ahead->elts; + + for (i = 0; i < ahead->nelts; i++) { + char *k = aheaders[i] = apr_pstrdup(ctx->pool, elts[i].key); + while(*k) { + *k = tolower(*k); + k++; + } + } + + header_gnome_sort(aheaders, ahead->nelts); + for(i=0; inelts; i++) { + canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); + } + canonical_request = apr_pstrcat(ctx->pool, canonical_request, "\n", NULL); + for(i=0; inelts; i++) { + if(i==ahead->nelts-1) { + canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],NULL); + } else { + canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],";",NULL); + } + } + canonical_request = apr_pstrcat(ctx->pool, canonical_request, "\n", sha1, NULL); + //printf("canonical request: %s\n",canonical_request); + + tosign = apr_pstrcat(ctx->pool, "AWS4-HMAC-SHA256\n",x_amz_date,"\n",NULL); + x_amz_date[8]=0; + sha256((unsigned char*)canonical_request, strlen(canonical_request), sha1); + sha_hex_encode(sha1,32); + tosign = apr_pstrcat(ctx->pool, tosign, x_amz_date, "/", s3->region, "/s3/aws4_request\n", sha1,NULL); + //printf("key to sign: %s\n",tosign); + + key = apr_pstrcat(ctx->pool, "AWS4", s3->secret, NULL); + hmac_sha256((unsigned char*)x_amz_date, 8, (unsigned char*)key, strlen(key), sha1, 32); + hmac_sha256((unsigned char*)s3->region, strlen(s3->region), sha1, 32, sha2, 32); + hmac_sha256((unsigned char*)"s3", 2, sha2, 32, sha1, 32); + hmac_sha256((unsigned char*)"aws4_request", 12, sha1, 32, sha2, 32); + hmac_sha256((unsigned char*)tosign, strlen(tosign), sha2, 32, sha1, 32); + sha_hex_encode(sha1,32); + + + auth = apr_pstrcat(ctx->pool, "AWS4-HMAC-SHA256 Credential=",s3->id,"/",x_amz_date,"/",s3->region,"/s3/aws4_request,SignedHeaders=",NULL); + + for(i=0; inelts; i++) { + if(i==ahead->nelts-1) { + auth = apr_pstrcat(ctx->pool, auth, aheaders[i],NULL); + } else { + auth = apr_pstrcat(ctx->pool, auth, aheaders[i],";",NULL); + } + } + auth = apr_pstrcat(ctx->pool, auth, ",Signature=", sha1, NULL); + + apr_table_set(headers, "Authorization", auth); +} + +static void _mapcache_cache_s3_put_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "PUT", pcache, tile, url, headers); +} +static void _mapcache_cache_s3_get_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "GET", pcache, tile, url, headers); +} +static void _mapcache_cache_s3_head_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "HEAD", pcache, tile, url, headers); +} +static void _mapcache_cache_s3_delete_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "DELETE", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_put_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "PUT", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_get_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "GET", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_head_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "HEAD", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_delete_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "DELETE", pcache, tile, url, headers); +} +static void _mapcache_cache_google_put_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "PUT", pcache, tile, url, headers); +} +static void _mapcache_cache_google_get_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "GET", pcache, tile, url, headers); +} +static void _mapcache_cache_google_head_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "HEAD", pcache, tile, url, headers); +} +static void _mapcache_cache_google_delete_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "DELETE", pcache, tile, url, headers); +} + + +static int _mapcache_cache_rest_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + int status; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.has_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.has_tile); + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.has_tile.add_headers) { + rcache->rest.has_tile.add_headers(ctx,rcache,tile,url,headers); + } + + status = _head_request(ctx, url, headers); + + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + + if( status == 200) + return MAPCACHE_TRUE; + else + return MAPCACHE_FALSE; +} + +static void _mapcache_cache_rest_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + int status; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.delete_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.delete_tile); + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.delete_tile.add_headers) { + rcache->rest.delete_tile.add_headers(ctx,rcache,tile,url,headers); + } + + status = _delete_request(ctx, url, headers); + GC_CHECK_ERROR(ctx); + + if(status!=200 && status!=202 && status!=204) { + //ctx->set_error(ctx,500,"rest delete returned code %d", status); + } +} + + +/** + * \brief get file content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored in the file + * \private \memberof mapcache_cache_rest + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_rest_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.get_tile, &url); + if(tile->allow_redirect && rcache->use_redirects) { + tile->redirect = url; + return MAPCACHE_SUCCESS; + } + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.get_tile); + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.get_tile.add_headers) { + rcache->rest.get_tile.add_headers(ctx,rcache,tile,url,headers); + } + tile->encoded_data = _get_request(ctx, url, headers); + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + + if(!tile->encoded_data) + return MAPCACHE_CACHE_MISS; + + return MAPCACHE_SUCCESS; +} + +static void _mapcache_cache_rest_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + CURL *curl = curl_easy_init(); + int i; + + if(!curl) { + ctx->set_error(ctx,500,"failed to create curl handle"); + return; + } + + for(i=0; irest, &rcache->rest.set_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.set_tile); + + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + if(GC_HAS_ERROR(ctx)) { + goto multi_put_cleanup; + } + } + + apr_table_set(headers,"Content-Length",apr_psprintf(ctx->pool,"%lu",tile->encoded_data->size)); + if(tile->tileset->format && tile->tileset->format->mime_type) + apr_table_set(headers, "Content-Type", tile->tileset->format->mime_type); + else { + mapcache_image_format_type imgfmt = mapcache_imageio_header_sniff(ctx,tile->encoded_data); + if(imgfmt == GC_JPEG) { + apr_table_set(headers, "Content-Type", "image/jpeg"); + } else if (imgfmt == GC_PNG) { + apr_table_set(headers, "Content-Type", "image/png"); + } + } + + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.set_tile.add_headers) { + rcache->rest.set_tile.add_headers(ctx,rcache,tile,url,headers); + } + _put_request(ctx, curl, tile->encoded_data, url, headers); + if(GC_HAS_ERROR(ctx)) { + goto multi_put_cleanup; + } + } + +multi_put_cleanup: + /* always cleanup */ + curl_easy_cleanup(curl); + +} + +/** + * \brief write tile data to rest backend + * + * writes the content of mapcache_tile::data to disk. + * \returns MAPCACHE_FAILURE if there is no data to write, or if the tile isn't locked + * \returns MAPCACHE_SUCCESS if the tile has been successfully written to disk + * \private \memberof mapcache_cache_rest + * \sa mapcache_cache::tile_set() + */ +static void _mapcache_cache_rest_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + return _mapcache_cache_rest_multi_set(ctx, pcache, tile, 1); +} + + +static void _mapcache_cache_rest_operation_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_rest_operation *op) +{ + ezxml_t cur_node; + if ((cur_node = ezxml_child(node,"headers")) != NULL) { + ezxml_t header_node; + op->headers = apr_table_make(ctx->pool,3); + for(header_node = cur_node->child; header_node; header_node = header_node->sibling) { + apr_table_set(op->headers, header_node->name, header_node->txt); + } + } +} + +/** + * \private \memberof mapcache_cache_rest + */ +static void _mapcache_cache_rest_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_rest *dcache = (mapcache_cache_rest*)cache; + if ((cur_node = ezxml_child(node,"url")) != NULL) { + dcache->rest.tile_url = apr_pstrdup(ctx->pool,cur_node->txt); + } + if ((cur_node = ezxml_child(node,"use_redirects")) != NULL) { + if(!strcasecmp(cur_node->txt,"true")) { + dcache->use_redirects = 1; + } + } + if ((cur_node = ezxml_child(node,"headers")) != NULL) { + ezxml_t header_node; + dcache->rest.common_headers = apr_table_make(ctx->pool,3); + for(header_node = cur_node->child; header_node; header_node = header_node->sibling) { + apr_table_set(dcache->rest.common_headers, header_node->name, header_node->txt); + } + } + for(cur_node = ezxml_child(node,"operation"); cur_node; cur_node = cur_node->next) { + char *type = (char*)ezxml_attr(cur_node,"type"); + if(!type) { + ctx->set_error(ctx,400," with no \"type\" attribute in cache (%s)", cache->name); + return; + } + if(!strcasecmp(type,"put")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.set_tile); + GC_CHECK_ERROR(ctx); + } else if(!strcasecmp(type,"get")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.get_tile); + GC_CHECK_ERROR(ctx); + } else if(!strcasecmp(type,"head")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.has_tile); + GC_CHECK_ERROR(ctx); + } else if(!strcasecmp(type,"delete")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.delete_tile); + GC_CHECK_ERROR(ctx); + } else { + ctx->set_error(ctx,400," with unknown \"type\" (%s) attribute in cache (%s) (expecting put, get, head or delete)", type, cache->name); + return; + } + } + +} + +static void _mapcache_cache_google_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_google *google = (mapcache_cache_google*)cache; + _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); + GC_CHECK_ERROR(ctx); + if ((cur_node = ezxml_child(node,"access")) != NULL) { + google->access = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"google cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + google->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"google cache (%s) is missing required child", cache->name); + return; + } +} + +static void _mapcache_cache_s3_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_s3 *s3 = (mapcache_cache_s3*)cache; + _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); + GC_CHECK_ERROR(ctx); + if ((cur_node = ezxml_child(node,"id")) != NULL) { + s3->id = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + s3->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"region")) != NULL) { + s3->region = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child", cache->name); + return; + } +} + +static void _mapcache_cache_azure_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_azure *azure = (mapcache_cache_azure*)cache; + _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); + GC_CHECK_ERROR(ctx); + if ((cur_node = ezxml_child(node,"id")) != NULL) { + azure->id = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"azure cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + azure->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"azure cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"container")) != NULL) { + azure->container = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"azure cache (%s) is missing required child", cache->name); + return; + } +} + +/** + * \private \memberof mapcache_cache_rest + */ +static void _mapcache_cache_rest_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ + mapcache_cache_rest *dcache = (mapcache_cache_rest*)cache; + + + if(!dcache->rest.tile_url) { + if(!dcache->rest.delete_tile.tile_url) { + ctx->set_error(ctx,400, "rest cache (%s) has no global and no for delete_tile operation", cache->name); + return; + } + if(!dcache->rest.get_tile.tile_url) { + ctx->set_error(ctx,400, "rest cache (%s) has no global and no for get_tile operation", cache->name); + return; + } + if(!dcache->rest.set_tile.tile_url) { + ctx->set_error(ctx,400, "rest cache (%s) has no global and no for set_tile operation", cache->name); + return; + } + } +} + +void mapcache_cache_rest_init(mapcache_context *ctx, mapcache_cache_rest *cache) { + cache->retry_count = 0; + cache->use_redirects = 0; + cache->rest.get_tile.method = MAPCACHE_REST_METHOD_GET; + cache->rest.set_tile.method = MAPCACHE_REST_METHOD_PUT; + cache->rest.delete_tile.method = MAPCACHE_REST_METHOD_DELETE; + cache->rest.multi_set_tile.method = MAPCACHE_REST_METHOD_PUT; + cache->rest.has_tile.method = MAPCACHE_REST_METHOD_HEAD; + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_REST; + cache->cache.tile_delete = _mapcache_cache_rest_delete; + cache->cache.tile_get = _mapcache_cache_rest_get; + cache->cache.tile_exists = _mapcache_cache_rest_has_tile; + cache->cache.tile_set = _mapcache_cache_rest_set; + cache->cache.tile_multi_set = _mapcache_cache_rest_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_rest_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_rest_configuration_parse_xml; +} +/** + * \brief creates and initializes a mapcache_rest_cache + */ +mapcache_cache* mapcache_cache_rest_create(mapcache_context *ctx) +{ + mapcache_cache_rest *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_rest)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate rest cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,cache); + cache->provider = MAPCACHE_REST_PROVIDER_NONE; + return (mapcache_cache*)cache; +} + +/** + * \brief creates and initializes a mapcache_s3_cache + */ +mapcache_cache* mapcache_cache_s3_create(mapcache_context *ctx) +{ + mapcache_cache_s3 *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_s3)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate s3 cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,&cache->cache); + cache->cache.provider = MAPCACHE_REST_PROVIDER_S3; + cache->cache.cache.configuration_parse_xml = _mapcache_cache_s3_configuration_parse_xml; + cache->cache.rest.get_tile.add_headers = _mapcache_cache_s3_get_headers_add; + cache->cache.rest.has_tile.add_headers = _mapcache_cache_s3_head_headers_add; + cache->cache.rest.set_tile.add_headers = _mapcache_cache_s3_put_headers_add; + cache->cache.rest.delete_tile.add_headers = _mapcache_cache_s3_delete_headers_add; + return (mapcache_cache*)cache; +} + +/** + * \brief creates and initializes a mapcache_azure_cache + */ +mapcache_cache* mapcache_cache_azure_create(mapcache_context *ctx) +{ + mapcache_cache_azure *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_azure)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate azure cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,&cache->cache); + cache->cache.provider = MAPCACHE_REST_PROVIDER_AZURE; + cache->cache.cache.configuration_parse_xml = _mapcache_cache_azure_configuration_parse_xml; + cache->cache.rest.get_tile.add_headers = _mapcache_cache_azure_get_headers_add; + cache->cache.rest.has_tile.add_headers = _mapcache_cache_azure_head_headers_add; + cache->cache.rest.set_tile.add_headers = _mapcache_cache_azure_put_headers_add; + cache->cache.rest.delete_tile.add_headers = _mapcache_cache_azure_delete_headers_add; + return (mapcache_cache*)cache; +} + +/** + * \brief creates and initializes a mapcache_google_cache + */ +mapcache_cache* mapcache_cache_google_create(mapcache_context *ctx) +{ + mapcache_cache_google *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_google)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate google cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,&cache->cache); + cache->cache.provider = MAPCACHE_REST_PROVIDER_GOOGLE; + cache->cache.cache.configuration_parse_xml = _mapcache_cache_google_configuration_parse_xml; + cache->cache.rest.get_tile.add_headers = _mapcache_cache_google_get_headers_add; + cache->cache.rest.has_tile.add_headers = _mapcache_cache_google_head_headers_add; + cache->cache.rest.set_tile.add_headers = _mapcache_cache_google_put_headers_add; + cache->cache.rest.delete_tile.add_headers = _mapcache_cache_google_delete_headers_add; + return (mapcache_cache*)cache; +} + + + +/* vim: ts=2 sts=2 et sw=2 +*/ diff -Nru mapcache-1.2.1/lib/cache_riak.c mapcache-1.4.1/lib/cache_riak.c --- mapcache-1.2.1/lib/cache_riak.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/cache_riak.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,462 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching support file: riak cache backend. + * Author: Michael Downey and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2013 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache-config.h" +#ifdef USE_RIAK + +#include "mapcache.h" + +#include +#include +#include + +#include +#include + +/* + * Since we don't construct the connection pool and store it in the cache object + * we have to store all the connections in a hash map in case there are multiple + * riak caches defined. + */ + +struct riak_conn_params { + mapcache_cache_riak *cache; +}; + +void mapcache_riak_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + mapcache_cache_riak *cache = ((struct riak_conn_params*)params)->cache; + struct RIACK_CONNECTION_OPTIONS options; + struct RIACK_CLIENT *client = riack_new_client(0); + + if (client == NULL) { + ctx->set_error(ctx,500,"failed to riack_new_client(0)"); + return; + } + + options.recv_timeout_ms = 2000; + options.send_timeout_ms = 2000; + if (riack_connect(client, cache->host, cache->port, &options) != RIACK_SUCCESS) { + riack_free(client); + ctx->set_error(ctx,500,"failed to riack_connect()"); + return; + } + + if (riack_ping(client) != RIACK_SUCCESS) { + riack_free(client); + ctx->set_error(ctx,500,"failed to riack_ping()"); + return; + } + + *conn_ = client; +} + +void mapcache_riak_connection_destructor(void *conn_) { + struct RIACK_CLIENT *client = (struct RIACK_CLIENT *)conn_; + riack_free(client); +} + +static mapcache_pooled_connection* _riak_get_connection(mapcache_context *ctx, mapcache_cache_riak *cache, mapcache_tile *tile) +{ + mapcache_pooled_connection *pc; + struct riak_conn_params params; + + params.cache = cache; + + pc = mapcache_connection_pool_get_connection(ctx,cache->cache.name,mapcache_riak_connection_constructor, + mapcache_riak_connection_destructor, ¶ms); + + return pc; +} + +static int _mapcache_cache_riak_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + int error; + int retries = 3; + RIACK_STRING key; + struct RIACK_GET_OBJECT obj; + struct RIACK_CLIENT *client; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + key.value = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FALSE; + } + key.len = strlen(key.value); + + pc = _riak_get_connection(ctx, cache, tile); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FALSE; + } + client = pc->connection; + + do + { + error = riack_get(client, cache->bucket, key, 0, &obj); + if (error != RIACK_SUCCESS) { + ctx->log(ctx, MAPCACHE_WARN, "Retry %d in riak_has_tile for tile %s from cache %s due to error %d", (4-retries), key.value, cache->cache.name, error); + for (error = riack_reconnect(client); + error != RIACK_SUCCESS && retries > 0; + error = riack_reconnect(client)) + { + --retries; + } + + --retries; + } + } + while (error != RIACK_SUCCESS && retries >= 0); + + if (error != RIACK_SUCCESS) { + riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. + mapcache_connection_pool_invalidate_connection(ctx,pc); + ctx->set_error(ctx, 500, "riak: failed to get key %s: %d", key, error); + return MAPCACHE_FALSE; + } + + if (obj.object.content_count < 1 || obj.object.content[0].data_len == 0) { + error = MAPCACHE_FALSE; + } else { + error = MAPCACHE_TRUE; + } + + riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. + mapcache_connection_pool_release_connection(ctx,pc); + + return error; +} + +static void _mapcache_cache_riak_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + int error; + RIACK_STRING key; + struct RIACK_CLIENT *client; + struct RIACK_DEL_PROPERTIES properties; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + memset(&properties, 0, sizeof(struct RIACK_DEL_PROPERTIES)); + + + key.value = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + GC_CHECK_ERROR(ctx); + key.len = strlen(key.value); + + pc = _riak_get_connection(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + client = pc->connection; + + properties.rw_use = 1; + properties.rw = (4294967295 - 3); // Special value meaning "ALL" + error = riack_delete(client, cache->bucket, key, &properties); + + mapcache_connection_pool_release_connection(ctx,pc); + + if (error != RIACK_SUCCESS) { + ctx->set_error(ctx, 500, "riak: failed to delete key %s: %d", key, error); + } +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the riak server + * \private \memberof mapcache_cache_riak + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_riak_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + int error; + int connect_error = RIACK_SUCCESS; + int retries = 3; + RIACK_STRING key; + struct RIACK_GET_OBJECT obj; + struct RIACK_GET_PROPERTIES properties; + struct RIACK_CLIENT *client; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + memset(&properties, 0, sizeof(struct RIACK_GET_PROPERTIES)); + + //Use Buckets defaults instead of setting the read/write attributes + /* + properties.r_use = 1; + properties.r = 1; + */ + + + key.value = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + key.len = strlen(key.value); + + tile->encoded_data = mapcache_buffer_create(0, ctx->pool); + + pc = _riak_get_connection(ctx, cache, tile); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + client = pc->connection; + + // If we get an error it is advised that we call reconnect. It also appears + // that every now and then we get an error and need to retry once again to + // get it to work. + do + { + error = riack_get(client, cache->bucket, key, &properties, &obj); + if (error != RIACK_SUCCESS) { + ctx->log(ctx, MAPCACHE_WARN, "Retry %d in riak_get for tile %s from cache %s due to error %d", (4-retries), key.value, cache->cache.name, error); + for (connect_error = riack_reconnect(client); + connect_error != RIACK_SUCCESS && retries > 0; + connect_error = riack_reconnect(client)) + { + --retries; + } + + --retries; + } + } + while (error != RIACK_SUCCESS && retries >= 0); + + if (error != RIACK_SUCCESS) + { + if (connect_error != RIACK_SUCCESS) + mapcache_connection_pool_invalidate_connection(ctx,pc); + else + mapcache_connection_pool_release_connection(ctx,pc); + + ctx->set_error(ctx, 500, "Failed to get tile %s from cache %s due to error %d", key.value, cache->cache.name, error); + return MAPCACHE_FAILURE; + } + + // Check if tile exists. If it doesn't we need to return CACHE_MISS or things go wrong. + // Mapcache doesn't appear to use the has_tile function and uses _get instead so we need + // to do this sort of test here instead of erroring. + if (obj.object.content_count < 1 || obj.object.content[0].data_len == 0) { + riack_free_get_object(client, &obj); // Need to free the object here as well. + mapcache_connection_pool_release_connection(ctx,pc); + return MAPCACHE_CACHE_MISS; + } + + // Copy the data into the buffer + mapcache_buffer_append(tile->encoded_data, obj.object.content[0].data_len, obj.object.content[0].data); + + riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. + + mapcache_connection_pool_release_connection(ctx,pc); + + return MAPCACHE_SUCCESS; +} + +/** + * \brief push tile data to riak + * + * writes the content of mapcache_tile::data to the configured riak instance(s) + * \private \memberof mapcache_cache_riak + * \sa mapcache_cache::tile_set() + */ +static void _mapcache_cache_riak_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + char *key,*content_type; + int error; + int connect_error = RIACK_SUCCESS; + int retries = 3; + struct RIACK_OBJECT object; + struct RIACK_CONTENT content; + struct RIACK_PUT_PROPERTIES properties; + struct RIACK_CLIENT *client; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + memset(&content, 0, sizeof(struct RIACK_CONTENT)); + memset(&object, 0, sizeof(struct RIACK_OBJECT)); + memset(&properties, 0, sizeof(struct RIACK_PUT_PROPERTIES)); + + //Use Buckets defaults instead of setting the read/write attributes + /* + properties.w_use = 1; + properties.w = 1; + + properties.dw_use = 1; + properties.dw = 0;*/ + + + key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + GC_CHECK_ERROR(ctx); + + if (!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + content_type = tile->tileset->format?(tile->tileset->format->mime_type?tile->tileset->format->mime_type:NULL):NULL; + + if(!content_type) { + /* compute the content-type */ + mapcache_image_format_type t = mapcache_imageio_header_sniff(ctx,tile->encoded_data); + if(t == GC_PNG) + content_type = "image/png"; + else if(t == GC_JPEG) + content_type = "image/jpeg"; + } + + pc = _riak_get_connection(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + client = pc->connection; + + // Set up the riak object to put. Need to do this after we get the client connection + object.bucket.value = cache->bucket.value; + object.bucket.len = cache->bucket.len; + object.key.value = key; + object.key.len = strlen(key); + object.vclock.len = 0; + object.content_count = 1; + object.content = &content; + content.content_type.value = content_type; + content.content_type.len = content_type?strlen(content_type):0; + content.data = (uint8_t*)tile->encoded_data->buf; + content.data_len = tile->encoded_data->size; + + // If we get an error it is advised that we call reconnect. It also appears + // that every now and then we get an error and need to retry once again to + // get it to work. + do + { + error = riack_put(client, object, 0, &properties); + if (error != RIACK_SUCCESS) { + ctx->log(ctx, MAPCACHE_WARN, "Retry %d in riak_set for tile %s from cache %s due to eror %d", (4 - retries), key, cache->cache.name, error); + for (connect_error = riack_reconnect(client); + connect_error != RIACK_SUCCESS && retries > 0; + connect_error = riack_reconnect(client)) + { + --retries; + } + + --retries; + } + } + while (error != RIACK_SUCCESS && retries >= 0); + + if (connect_error != RIACK_SUCCESS) + mapcache_connection_pool_invalidate_connection(ctx,pc); + else + mapcache_connection_pool_release_connection(ctx,pc); + + if (error != RIACK_SUCCESS) + { + ctx->set_error(ctx, 500, "failed to store tile %s to cache %s due to error %d.", key, cache->cache.name, error); + } +} + +/** + * \private \memberof mapcache_cache_riak + */ +static void _mapcache_cache_riak_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { + ezxml_t cur_node,xhost,xport,xbucket; + mapcache_cache_riak *dcache = (mapcache_cache_riak*)cache; + int servercount = 0; + + for (cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { + servercount++; + } + + if (!servercount) { + ctx->set_error(ctx, 400, "riak cache %s has no s configured", cache->name); + return; + } + + if (servercount > 1) { + ctx->set_error(ctx, 400, "riak cache %s has more than 1 server configured", cache->name); + return; + } + + cur_node = ezxml_child(node, "server"); + xhost = ezxml_child(cur_node, "host"); /* Host should contain just server */ + xport = ezxml_child(cur_node, "port"); + xbucket = ezxml_child(cur_node, "bucket"); + + if (!xhost || !xhost->txt || ! *xhost->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->host = apr_pstrdup(ctx->pool, xhost->txt); + if (dcache->host == NULL) { + ctx->set_error(ctx, 400, "cache %s: failed to allocate host string!", cache->name); + return; + } + } + + if (!xport || !xport->txt || ! *xport->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->port = atoi(xport->txt); + } + + if (!xbucket || !xbucket->txt || ! *xbucket->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->bucket.value = apr_pstrdup(ctx->pool, xbucket->txt); + if (dcache->bucket.value == NULL) { + ctx->set_error(ctx, 400, "cache %s: failed to allocate bucket string!", cache->name); + return; + } + dcache->bucket.len = strlen(dcache->bucket.value); + } +} + +/** + * \private \memberof mapcache_cache_riak + */ +static void _mapcache_cache_riak_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, mapcache_cfg *cfg) { + riack_init(); +} + +/** + * \brief creates and initializes a mapcache_riak_cache + */ +mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx) { + mapcache_cache_riak *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_riak)); + if (!cache) { + ctx->set_error(ctx, 500, "failed to allocate riak cache"); + return NULL; + } + + cache->cache.metadata = apr_table_make(ctx->pool, 3); + cache->cache.type = MAPCACHE_CACHE_RIAK; + cache->cache.tile_get = _mapcache_cache_riak_get; + cache->cache.tile_exists = _mapcache_cache_riak_has_tile; + cache->cache.tile_set = _mapcache_cache_riak_set; + cache->cache.tile_delete = _mapcache_cache_riak_delete; + cache->cache.configuration_parse_xml = _mapcache_cache_riak_configuration_parse_xml; + cache->cache.configuration_post_config = _mapcache_cache_riak_configuration_post_config; + cache->host = NULL; + cache->port = 8087; // Default RIAK port used for protobuf + + return (mapcache_cache*)cache; +} + +#endif diff -Nru mapcache-1.2.1/lib/cache_sqlite.c mapcache-1.4.1/lib/cache_sqlite.c --- mapcache-1.2.1/lib/cache_sqlite.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/cache_sqlite.c 2016-02-25 15:47:16.000000000 +0000 @@ -47,17 +47,20 @@ #include -static apr_hash_t *ro_connection_pools = NULL; -static apr_hash_t *rw_connection_pools = NULL; +struct sqlite_conn_params { + mapcache_cache_sqlite *cache; + char *dbfile; + int readonly; +}; struct sqlite_conn { sqlite3 *handle; - int readonly; int nstatements; sqlite3_stmt **prepared_statements; - char *errmsg; }; +#define SQLITE_CONN(pooled_connection) ((struct sqlite_conn*)((pooled_connection)->connection)) + #define HAS_TILE_STMT_IDX 0 #define GET_TILE_STMT_IDX 1 #define SQLITE_SET_TILE_STMT_IDX 2 @@ -70,8 +73,9 @@ #define MBTILES_DEL_TILE_STMT1_IDX 7 #define MBTILES_DEL_TILE_STMT2_IDX 8 +static void _mapcache_cache_sqlite_filename_for_tile(mapcache_context *ctx, mapcache_cache_sqlite *dcache, mapcache_tile *tile, char **path); -static int _sqlite_set_pragmas(apr_pool_t *pool, mapcache_cache_sqlite* cache, struct sqlite_conn *conn) +static void _sqlite_set_pragmas(mapcache_context *ctx, mapcache_cache_sqlite* cache, struct sqlite_conn *conn) { if (cache->pragmas && !apr_is_empty_table(cache->pragmas)) { const apr_array_header_t *elts = apr_table_elts(cache->pragmas); @@ -80,7 +84,7 @@ char *pragma_stmt; for (i = 0; i < elts->nelts; i++) { apr_table_entry_t entry = APR_ARRAY_IDX(elts, i, apr_table_entry_t); - pragma_stmt = apr_psprintf(pool,"PRAGMA %s=%s",entry.key,entry.val); + pragma_stmt = apr_psprintf(ctx->pool,"PRAGMA %s=%s",entry.key,entry.val); do { ret = sqlite3_exec(conn->handle, pragma_stmt, 0, 0, NULL); if (ret != SQLITE_OK && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { @@ -88,78 +92,61 @@ } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret != SQLITE_OK) { - conn->errmsg = apr_psprintf(pool,"failed to execute pragma statement %s",pragma_stmt); - return MAPCACHE_FAILURE; + ctx->set_error(ctx, 500, "failed to execute pragma statement %s",pragma_stmt); + return; } } } - return MAPCACHE_SUCCESS; + return; +} + + + +static void mapcache_sqlite_release_conn(mapcache_context *ctx, mapcache_pooled_connection *conn) { + mapcache_connection_pool_release_connection(ctx,conn); } -static apr_status_t _sqlite_reslist_get_rw_connection(void **conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; int flags; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) params; - struct sqlite_conn *conn = apr_pcalloc(pool, sizeof (struct sqlite_conn)); + struct sqlite_conn_params *sq_params = (struct sqlite_conn_params*)params; + struct sqlite_conn *conn = calloc(1, sizeof (struct sqlite_conn)); *conn_ = conn; - flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_CREATE; - ret = sqlite3_open_v2(cache->dbfile, &conn->handle, flags, NULL); + if(sq_params->readonly) { + flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; + } else { + flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_CREATE; + mapcache_make_parent_dirs(ctx,sq_params->dbfile); + GC_CHECK_ERROR(ctx); + } + ret = sqlite3_open_v2(sq_params->dbfile, &conn->handle, flags, NULL); if (ret != SQLITE_OK) { - conn->errmsg = apr_psprintf(pool,"sqlite backend failed to open db %s: %s", cache->dbfile, sqlite3_errmsg(conn->handle)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"sqlite backend failed to open db %s: %s", sq_params->dbfile, sqlite3_errmsg(conn->handle)); + return; } sqlite3_busy_timeout(conn->handle, 300000); do { - ret = sqlite3_exec(conn->handle, cache->create_stmt.sql, 0, 0, NULL); + ret = sqlite3_exec(conn->handle, sq_params->cache->create_stmt.sql, 0, 0, NULL); if (ret != SQLITE_OK && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { break; } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret != SQLITE_OK) { - conn->errmsg = apr_psprintf(pool, "sqlite backend failed to create db schema on %s: %s", cache->dbfile, sqlite3_errmsg(conn->handle)); - sqlite3_close(conn->handle); - return APR_EGENERAL; - } - conn->readonly = 0; - ret = _sqlite_set_pragmas(pool, cache, conn); - if(ret != MAPCACHE_SUCCESS) { + ctx->set_error(ctx,500, "sqlite backend failed to create db schema on %s: %s", sq_params->dbfile, sqlite3_errmsg(conn->handle)); sqlite3_close(conn->handle); - return APR_EGENERAL; - } - conn->prepared_statements = calloc(cache->n_prepared_statements,sizeof(sqlite3_stmt*)); - conn->nstatements = cache->n_prepared_statements; - - return APR_SUCCESS; -} - -static apr_status_t _sqlite_reslist_get_ro_connection(void **conn_, void *params, apr_pool_t *pool) -{ - int ret; - int flags; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) params; - struct sqlite_conn *conn = apr_pcalloc(pool, sizeof (struct sqlite_conn)); - *conn_ = conn; - flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; - ret = sqlite3_open_v2(cache->dbfile, &conn->handle, flags, NULL); - - if (ret != SQLITE_OK) { - return APR_EGENERAL; + return; } - sqlite3_busy_timeout(conn->handle, 300000); - conn->readonly = 1; - - ret = _sqlite_set_pragmas(pool,cache, conn); - if (ret != MAPCACHE_SUCCESS) { + _sqlite_set_pragmas(ctx, sq_params->cache, conn); + if(GC_HAS_ERROR(ctx)) { sqlite3_close(conn->handle); - return APR_EGENERAL; + return; } - conn->prepared_statements = calloc(cache->n_prepared_statements,sizeof(sqlite3_stmt*)); - conn->nstatements = cache->n_prepared_statements; - return APR_SUCCESS; + conn->prepared_statements = calloc(sq_params->cache->n_prepared_statements,sizeof(sqlite3_stmt*)); + conn->nstatements = sq_params->cache->n_prepared_statements; } -static apr_status_t _sqlite_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_connection_destructor(void *conn_) { struct sqlite_conn *conn = (struct sqlite_conn*) conn_; int i; @@ -170,109 +157,106 @@ } free(conn->prepared_statements); sqlite3_close(conn->handle); - return APR_SUCCESS; } -static struct sqlite_conn* _sqlite_get_conn(mapcache_context *ctx, mapcache_tile* tile, int readonly) { - apr_status_t rv; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = NULL; - apr_reslist_t *pool = NULL; - apr_hash_t *pool_container; - if (readonly) { - pool_container = ro_connection_pools; +static mapcache_pooled_connection* mapcache_sqlite_get_conn(mapcache_context *ctx, mapcache_cache_sqlite *cache, mapcache_tile *tile, int readonly) { + struct sqlite_conn_params params; + mapcache_pooled_connection *pc; + char *key; + _mapcache_cache_sqlite_filename_for_tile(ctx,cache,tile,¶ms.dbfile); + params.cache = cache; + params.readonly = readonly; + if(!strstr(cache->dbfile,"{")) { + key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",cache->cache.name,NULL); } else { - pool_container = rw_connection_pools; - } - if(!pool_container || NULL == (pool = apr_hash_get(pool_container,cache->cache.name, APR_HASH_KEY_STRING)) ) { -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(!ro_connection_pools) { - ro_connection_pools = apr_hash_make(ctx->process_pool); - rw_connection_pools = apr_hash_make(ctx->process_pool); - } - - /* probably doesn't exist, unless the previous mutex locked us, so we check */ - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - if(!pool) { - /* there where no existing connection pools, create them*/ - rv = apr_reslist_create(&pool, - 0 /* min */, - 10 /* soft max */, - 200 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _sqlite_reslist_get_ro_connection, /* resource constructor */ - _sqlite_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb ro connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(ro_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - rv = apr_reslist_create(&pool, - 0 /* min */, - 1 /* soft max */, - 1 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _sqlite_reslist_get_rw_connection, /* resource constructor */ - _sqlite_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb rw connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(rw_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - } -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(readonly) - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - else - pool = apr_hash_get(rw_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - assert(pool); - } - rv = apr_reslist_acquire(pool, (void **) &conn); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to aquire connection to sqlite backend: %s", (conn && conn->errmsg)?conn->errmsg:"unknown error"); - return NULL; + key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",params.dbfile,NULL); } - return conn; + pc = mapcache_connection_pool_get_connection(ctx,key,mapcache_sqlite_connection_constructor,mapcache_sqlite_connection_destructor,¶ms); + return pc; } -static void _sqlite_release_conn(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn) + +/** + * \brief return sqlite db filename for given tile + * + * \param tile the tile to get the key from + * \param path pointer to a char* that will contain the filename + * \param r + * \private \memberof mapcache_cache_sqlite + */ +static void _mapcache_cache_sqlite_filename_for_tile(mapcache_context *ctx, mapcache_cache_sqlite *dcache, mapcache_tile *tile, char **path) { - apr_reslist_t *pool; - apr_hash_t *pool_container; - if(conn->readonly) { - pool_container = ro_connection_pools; - } else { - pool_container = rw_connection_pools; + *path = dcache->dbfile; + + if(strstr(*path,"{")) { + /* + * generic template substitutions + */ + if(strstr(*path,"{tileset}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{tileset}", + tile->tileset->name); + if(strstr(*path,"{grid}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", + tile->grid_link->grid->name); + if(tile->dimensions && strstr(*path,"{dim}")) { + char *dimstring=""; + const apr_array_header_t *elts = apr_table_elts(tile->dimensions); + int i = elts->nelts; + while(i--) { + apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); + const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->val,"/.",'#'); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",dimval,NULL); + } + *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); + } + + + while(strstr(*path,"{z}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{z}", + apr_psprintf(ctx->pool,dcache->z_fmt,tile->z)); + + if(dcache->count_x > 0) { + while(strstr(*path,"{div_x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{div_x}", + apr_psprintf(ctx->pool,dcache->div_x_fmt,tile->x/dcache->count_x)); + while(strstr(*path,"{inv_div_x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_x}", + apr_psprintf(ctx->pool,dcache->inv_div_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x)); + while(strstr(*path,"{x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{x}", + apr_psprintf(ctx->pool,dcache->x_fmt,tile->x/dcache->count_x*dcache->count_x)); + while(strstr(*path,"{inv_x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_x}", + apr_psprintf(ctx->pool,dcache->inv_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x*dcache->count_x)); + } + + if(dcache->count_y > 0) { + while(strstr(*path,"{div_y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{div_y}", + apr_psprintf(ctx->pool,dcache->div_y_fmt,tile->y/dcache->count_y)); + while(strstr(*path,"{inv_div_y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_y}", + apr_psprintf(ctx->pool,dcache->inv_div_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y)); + while(strstr(*path,"{y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{y}", + apr_psprintf(ctx->pool,dcache->y_fmt,tile->y/dcache->count_y*dcache->count_y)); + while(strstr(*path,"{inv_y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_y}", + apr_psprintf(ctx->pool,dcache->inv_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y*dcache->count_y)); + + } } - pool = apr_hash_get(pool_container,tile->tileset->cache->name, APR_HASH_KEY_STRING); - if (GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(pool, (void*) conn); - } else { - apr_reslist_release(pool, (void*) conn); + if(!*path) { + ctx->set_error(ctx,500, "failed to allocate tile key"); } } + /** * \brief apply appropriate tile properties to the sqlite statement */ -static void _bind_sqlite_params(mapcache_context *ctx, void *vstmt, mapcache_tile *tile) +static void _bind_sqlite_params(mapcache_context *ctx, void *vstmt, mapcache_cache_sqlite *cache, mapcache_tile *tile) { sqlite3_stmt *stmt = vstmt; int paramidx; @@ -311,8 +295,7 @@ paramidx = sqlite3_bind_parameter_index(stmt, ":data"); if (paramidx) { int written = 0; - if(((mapcache_cache_sqlite*)tile->tileset->cache)->detect_blank && tile->grid_link->grid->tile_sx == 256 && - tile->grid_link->grid->tile_sy == 256) { + if(cache->detect_blank) { if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); @@ -339,7 +322,7 @@ } } -static void _bind_mbtiles_params(mapcache_context *ctx, void *vstmt, mapcache_tile *tile) +static void _bind_mbtiles_params(mapcache_context *ctx, void *vstmt, mapcache_cache_sqlite *cache, mapcache_tile *tile) { sqlite3_stmt *stmt = vstmt; int paramidx; @@ -389,26 +372,29 @@ } -static int _mapcache_cache_sqlite_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_sqlite_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 1); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; sqlite3_stmt *stmt; int ret; + pc = mapcache_sqlite_get_conn(ctx,cache,tile,1); if (GC_HAS_ERROR(ctx)) { - if(conn) _sqlite_release_conn(ctx, tile, conn); + if(pc) mapcache_sqlite_release_conn(ctx, pc); if(!tile->tileset->read_only && tile->tileset->source) { /* not an error in this case, as the db file may not have been created yet */ ctx->clear_errors(ctx); } return MAPCACHE_FALSE; } + conn = SQLITE_CONN(pc); stmt = conn->prepared_statements[HAS_TILE_STMT_IDX]; if(!stmt) { sqlite3_prepare(conn->handle, cache->exists_stmt.sql, -1, &conn->prepared_statements[HAS_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[HAS_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW) { ctx->set_error(ctx, 500, "sqlite backend failed on has_tile: %s", sqlite3_errmsg(conn->handle)); @@ -419,46 +405,53 @@ ret = MAPCACHE_TRUE; } sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return ret; } -static void _mapcache_cache_sqlite_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_sqlite_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); - sqlite3_stmt *stmt = conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX]; + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; + sqlite3_stmt *stmt; int ret; + pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); if (GC_HAS_ERROR(ctx)) { - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } + conn = SQLITE_CONN(pc); + stmt = conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX]; if(!stmt) { sqlite3_prepare(conn->handle, cache->delete_stmt.sql, -1, &conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW) { ctx->set_error(ctx, 500, "sqlite backend failed on delete: %s", sqlite3_errmsg(conn->handle)); } sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_mbtiles_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_mbtiles_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; sqlite3_stmt *stmt1,*stmt2,*stmt3; int ret; const char *tile_id; size_t tile_id_size; + pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); if (GC_HAS_ERROR(ctx)) { - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } + conn = SQLITE_CONN(pc); stmt1 = conn->prepared_statements[MBTILES_DEL_TILE_SELECT_STMT_IDX]; stmt2 = conn->prepared_statements[MBTILES_DEL_TILE_STMT1_IDX]; stmt3 = conn->prepared_statements[MBTILES_DEL_TILE_STMT2_IDX]; @@ -474,19 +467,19 @@ /* first extract tile_id from the tile we will delete. We need this because we do not know * if the tile is empty or not. * If it is empty, we will not delete the image blob data from the images table */ - cache->bind_stmt(ctx, stmt1, tile); + cache->bind_stmt(ctx, stmt1, cache, tile); do { ret = sqlite3_step(stmt1); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { ctx->set_error(ctx, 500, "sqlite backend failed on mbtile del 1: %s", sqlite3_errmsg(conn->handle)); sqlite3_reset(stmt1); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret == SQLITE_DONE) { /* tile does not exist, ignore */ sqlite3_reset(stmt1); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } else { tile_id = (const char*) sqlite3_column_text(stmt1, 0); @@ -495,13 +488,13 @@ /* delete the tile from the "map" table */ - cache->bind_stmt(ctx,stmt2, tile); + cache->bind_stmt(ctx,stmt2, cache, tile); ret = sqlite3_step(stmt2); if (ret != SQLITE_DONE && ret != SQLITE_ROW) { ctx->set_error(ctx, 500, "sqlite backend failed on mbtile del 2: %s", sqlite3_errmsg(conn->handle)); sqlite3_reset(stmt1); sqlite3_reset(stmt2); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } @@ -517,7 +510,7 @@ sqlite3_reset(stmt1); sqlite3_reset(stmt2); sqlite3_reset(stmt3); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } } @@ -525,15 +518,14 @@ sqlite3_reset(stmt1); sqlite3_reset(stmt2); sqlite3_reset(stmt3); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _single_mbtile_set(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn) +static void _single_mbtile_set(mapcache_context *ctx, mapcache_cache_sqlite *cache, mapcache_tile *tile, struct sqlite_conn *conn) { sqlite3_stmt *stmt1,*stmt2; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)tile->tileset->cache; int ret; if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); @@ -552,8 +544,8 @@ stmt1 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT1_IDX]; stmt2 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT2_IDX]; } - cache->bind_stmt(ctx, stmt1, tile); - cache->bind_stmt(ctx, stmt2, tile); + cache->bind_stmt(ctx, stmt1, cache, tile); + cache->bind_stmt(ctx, stmt2, cache, tile); } else { stmt1 = conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX]; stmt2 = conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX]; @@ -567,8 +559,8 @@ stmt1 = conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX]; stmt2 = conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX]; } - cache->bind_stmt(ctx, stmt1, tile); - cache->bind_stmt(ctx, stmt2, tile); + cache->bind_stmt(ctx, stmt1, cache, tile); + cache->bind_stmt(ctx, stmt2, cache, tile); } do { ret = sqlite3_step(stmt1); @@ -596,47 +588,49 @@ sqlite3_reset(stmt2); } -static int _mapcache_cache_sqlite_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_sqlite_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; struct sqlite_conn *conn; sqlite3_stmt *stmt; int ret; - conn = _sqlite_get_conn(ctx, tile, 1); + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,tile,1); if (GC_HAS_ERROR(ctx)) { - if(conn) _sqlite_release_conn(ctx, tile, conn); if(tile->tileset->read_only || !tile->tileset->source) { + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_FAILURE; } else { /* not an error in this case, as the db file may not have been created yet */ ctx->clear_errors(ctx); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_CACHE_MISS; } } + conn = SQLITE_CONN(pc); stmt = conn->prepared_statements[GET_TILE_STMT_IDX]; if(!stmt) { sqlite3_prepare(conn->handle, cache->get_stmt.sql, -1, &conn->prepared_statements[GET_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[GET_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); do { ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { ctx->set_error(ctx, 500, "sqlite backend failed on get: %s", sqlite3_errmsg(conn->handle)); sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_FAILURE; } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret == SQLITE_DONE) { sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_CACHE_MISS; } else { const void *blob = sqlite3_column_blob(stmt, 0); int size = sqlite3_column_bytes(stmt, 0); if(size>0 && ((char*)blob)[0] == '#') { - tile->encoded_data = mapcache_empty_png_decode(ctx,blob,&tile->nodata); + tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy ,blob,&tile->nodata); } else { tile->encoded_data = mapcache_buffer_create(size, ctx->pool); memcpy(tile->encoded_data->buf, blob, size); @@ -647,14 +641,13 @@ apr_time_ansi_put(&(tile->mtime), mtime); } sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_SUCCESS; } } -static void _single_sqlitetile_set(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn) +static void _single_sqlitetile_set(mapcache_context *ctx, mapcache_cache_sqlite *cache, mapcache_tile *tile, struct sqlite_conn *conn) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; sqlite3_stmt *stmt = conn->prepared_statements[SQLITE_SET_TILE_STMT_IDX]; int ret; @@ -662,7 +655,7 @@ sqlite3_prepare(conn->handle, cache->set_stmt.sql, -1, &conn->prepared_statements[SQLITE_SET_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[SQLITE_SET_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); do { ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { @@ -676,29 +669,41 @@ sqlite3_reset(stmt); } -static void _mapcache_cache_sqlite_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_sqlite_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); - GC_CHECK_ERROR(ctx); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + struct sqlite_conn *conn; + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); - _single_sqlitetile_set(ctx,tile,conn); + _single_sqlitetile_set(ctx,cache, tile,conn); if (GC_HAS_ERROR(ctx)) { sqlite3_exec(conn->handle, "ROLLBACK TRANSACTION", 0, 0, 0); } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_sqlite_multi_set(mapcache_context *ctx, mapcache_tile *tiles, int ntiles) +static void _mapcache_cache_sqlite_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { - struct sqlite_conn *conn = _sqlite_get_conn(ctx, &tiles[0], 0); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; int i; - GC_CHECK_ERROR(ctx); + struct sqlite_conn *conn; + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,&tiles[0],0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); for (i = 0; i < ntiles; i++) { mapcache_tile *tile = &tiles[i]; - _single_sqlitetile_set(ctx,tile,conn); + _single_sqlitetile_set(ctx,cache, tile,conn); if(GC_HAS_ERROR(ctx)) break; } if (GC_HAS_ERROR(ctx)) { @@ -706,34 +711,42 @@ } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, &tiles[0], conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_mbtiles_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_mbtiles_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); - GC_CHECK_ERROR(ctx); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + struct sqlite_conn *conn; + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); if(GC_HAS_ERROR(ctx)) { - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } } sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); - _single_mbtile_set(ctx,tile,conn); + _single_mbtile_set(ctx, cache, tile,conn); if (GC_HAS_ERROR(ctx)) { sqlite3_exec(conn->handle, "ROLLBACK TRANSACTION", 0, 0, 0); } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_mbtiles_multi_set(mapcache_context *ctx, mapcache_tile *tiles, int ntiles) +static void _mapcache_cache_mbtiles_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { - struct sqlite_conn *conn = NULL; int i; + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; /* decode/encode image data before going into the sqlite write lock */ for (i = 0; i < ntiles; i++) { @@ -748,12 +761,17 @@ GC_CHECK_ERROR(ctx); } } - conn = _sqlite_get_conn(ctx, &tiles[0], 0); + pc = mapcache_sqlite_get_conn(ctx,cache,&tiles[0],0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); for (i = 0; i < ntiles; i++) { mapcache_tile *tile = &tiles[i]; - _single_mbtile_set(ctx,tile,conn); + _single_mbtile_set(ctx,cache,tile,conn); if(GC_HAS_ERROR(ctx)) break; } if (GC_HAS_ERROR(ctx)) { @@ -761,32 +779,69 @@ } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, &tiles[0], conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_sqlite_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_sqlite_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) { ezxml_t cur_node; - mapcache_cache_sqlite *dcache; + mapcache_cache_sqlite *cache; sqlite3_initialize(); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - dcache = (mapcache_cache_sqlite*) cache; + cache = (mapcache_cache_sqlite*) pcache; if ((cur_node = ezxml_child(node, "base")) != NULL) { ctx->set_error(ctx, 500, "sqlite config not supported anymore, use "); return; } if ((cur_node = ezxml_child(node, "dbname_template")) != NULL) { - ctx->set_error(ctx, 500, "sqlite config not supported anymore, use "); + ctx->set_error(ctx, 500, "sqlite config not supported anymore, use a \"multi-sqlite3\" cache type"); return; } if ((cur_node = ezxml_child(node, "dbfile")) != NULL) { - dcache->dbfile = apr_pstrdup(ctx->pool, cur_node->txt); + char *fmt; + cache->dbfile = apr_pstrdup(ctx->pool, cur_node->txt); + fmt = (char*)ezxml_attr(cur_node,"x_fmt"); + if(fmt && *fmt) { + cache->x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"y_fmt"); + if(fmt && *fmt) { + cache->y_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"z_fmt"); + if(fmt && *fmt) { + cache->z_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_x_fmt"); + if(fmt && *fmt) { + cache->inv_x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_y_fmt"); + if(fmt && *fmt) { + cache->inv_y_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"div_x_fmt"); + if(fmt && *fmt) { + cache->div_x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"div_y_fmt"); + if(fmt && *fmt) { + cache->div_y_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_div_x_fmt"); + if(fmt && *fmt) { + cache->inv_div_x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_div_y_fmt"); + if(fmt && *fmt) { + cache->inv_div_y_fmt = apr_pstrdup(ctx->pool,fmt); + } } - dcache->detect_blank = 0; + cache->detect_blank = 0; if ((cur_node = ezxml_child(node, "detect_blank")) != NULL) { - if(!strcasecmp(cur_node->txt,"true")) { - dcache->detect_blank = 1; + if(strcasecmp(cur_node->txt,"false")) { + cache->detect_blank = 1; } } @@ -796,20 +851,53 @@ } } if ((cur_node = ezxml_child(node, "pragma")) != NULL) { - dcache->pragmas = apr_table_make(ctx->pool,1); + cache->pragmas = apr_table_make(ctx->pool,1); while(cur_node) { char *name = (char*)ezxml_attr(cur_node,"name"); if(!name || !cur_node->txt || !strlen(cur_node->txt)) { ctx->set_error(ctx,500," missing name attribute"); return; } - apr_table_set(dcache->pragmas,name,cur_node->txt); + apr_table_set(cache->pragmas,name,cur_node->txt); cur_node = cur_node->next; } } - if (!dcache->dbfile) { - ctx->set_error(ctx, 500, "sqlite cache \"%s\" is missing entry", cache->name); - return; + if ((cur_node = ezxml_child(node, "queries")) != NULL) { + ezxml_t query_node; + if ((query_node = ezxml_child(cur_node, "exists")) != NULL) { + cache->exists_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "get")) != NULL) { + cache->get_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "set")) != NULL) { + cache->set_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "delete")) != NULL) { + cache->delete_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "create")) != NULL) { + cache->create_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + } + + cur_node = ezxml_child(node,"xcount"); + if(cur_node && cur_node->txt && *cur_node->txt) { + char *endptr; + cache->count_x = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0) { + ctx->set_error(ctx,400,"failed to parse xcount value %s for sqlite cache %s", cur_node->txt,cache->cache.name); + return; + } + } + cur_node = ezxml_child(node,"ycount"); + if(cur_node && cur_node->txt && *cur_node->txt) { + char *endptr; + cache->count_y = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0) { + ctx->set_error(ctx,400,"failed to parse ycount value %s for sqlite cache %s", cur_node->txt,cache->cache.name); + return; + } } } @@ -817,18 +905,24 @@ * \private \memberof mapcache_cache_sqlite */ static void _mapcache_cache_sqlite_configuration_post_config(mapcache_context *ctx, - mapcache_cache *cache, mapcache_cfg *cfg) + mapcache_cache *pcache, mapcache_cfg *cfg) { + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + if (!cache->dbfile) { + ctx->set_error(ctx, 500, "sqlite cache \"%s\" is missing entry", pcache->name); + return; + } } /** * \private \memberof mapcache_cache_sqlite */ static void _mapcache_cache_mbtiles_configuration_post_config(mapcache_context *ctx, - mapcache_cache *cache, mapcache_cfg *cfg) + mapcache_cache *pcache, mapcache_cfg *cfg) { /* check that only one tileset/grid references this cache, as mbtiles does not support multiple tilesets/grids per cache */ +#ifdef FIXME int nrefs = 0; apr_hash_index_t *tileseti = apr_hash_first(ctx->pool,cfg->tilesets); while(tileseti) { @@ -849,12 +943,9 @@ } tileseti = apr_hash_next(tileseti); } - +#endif } -/** - * \brief creates and initializes a mapcache_sqlite_cache - */ mapcache_cache* mapcache_cache_sqlite_create(mapcache_context *ctx) { mapcache_cache_sqlite *cache = apr_pcalloc(ctx->pool, sizeof (mapcache_cache_sqlite)); @@ -884,9 +975,15 @@ cache->n_prepared_statements = 4; cache->bind_stmt = _bind_sqlite_params; cache->detect_blank = 1; - return (mapcache_cache*) cache; + cache->x_fmt = cache->y_fmt = cache->z_fmt + = cache->inv_x_fmt = cache->inv_y_fmt + = cache->div_x_fmt = cache->div_y_fmt + = cache->inv_div_x_fmt = cache->inv_div_y_fmt = apr_pstrdup(ctx->pool,"%d"); + cache->count_x = cache->count_y = -1; + return (mapcache_cache*)cache; } + /** * \brief creates and initializes a mapcache_sqlite_cache */ diff -Nru mapcache-1.2.1/lib/cache_tiff.c mapcache-1.4.1/lib/cache_tiff.c --- mapcache-1.2.1/lib/cache_tiff.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/cache_tiff.c 2016-02-25 15:47:16.000000000 +0000 @@ -62,10 +62,9 @@ * \param r * \private \memberof mapcache_cache_tiff */ -static void _mapcache_cache_tiff_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_tiff_tile_key(mapcache_context *ctx, mapcache_cache_tiff *cache, mapcache_tile *tile, char **path) { - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)tile->tileset->cache; - *path = dcache->filename_template; + *path = cache->filename_template; /* * generic template substitutions @@ -91,23 +90,23 @@ while(strstr(*path,"{z}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{z}", - apr_psprintf(ctx->pool,dcache->z_fmt,tile->z)); + apr_psprintf(ctx->pool,cache->z_fmt,tile->z)); /* * x and y replacing, when the tiff files are numbered with an increasing * x,y scheme (adjacent tiffs have x-x'=1 or y-y'=1 */ while(strstr(*path,"{div_x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{div_x}", - apr_psprintf(ctx->pool,dcache->div_x_fmt,tile->x/dcache->count_x)); + apr_psprintf(ctx->pool,cache->div_x_fmt,tile->x/cache->count_x)); while(strstr(*path,"{div_y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{div_y}", - apr_psprintf(ctx->pool,dcache->div_y_fmt,tile->y/dcache->count_y)); + apr_psprintf(ctx->pool,cache->div_y_fmt,tile->y/cache->count_y)); while(strstr(*path,"{inv_div_y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_y}", - apr_psprintf(ctx->pool,dcache->inv_div_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y)); + apr_psprintf(ctx->pool,cache->inv_div_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/cache->count_y)); while(strstr(*path,"{inv_div_x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_x}", - apr_psprintf(ctx->pool,dcache->inv_div_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x)); + apr_psprintf(ctx->pool,cache->inv_div_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/cache->count_x)); /* * x and y replacing, when the tiff files are numbered with the index @@ -116,25 +115,24 @@ */ while(strstr(*path,"{x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{x}", - apr_psprintf(ctx->pool,dcache->x_fmt,tile->x/dcache->count_x*dcache->count_x)); + apr_psprintf(ctx->pool,cache->x_fmt,tile->x/cache->count_x*cache->count_x)); while(strstr(*path,"{y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{y}", - apr_psprintf(ctx->pool,dcache->y_fmt,tile->y/dcache->count_y*dcache->count_y)); + apr_psprintf(ctx->pool,cache->y_fmt,tile->y/cache->count_y*cache->count_y)); while(strstr(*path,"{inv_y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_y}", - apr_psprintf(ctx->pool,dcache->inv_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y*dcache->count_y)); + apr_psprintf(ctx->pool,cache->inv_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/cache->count_y*cache->count_y)); while(strstr(*path,"{inv_x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_x}", - apr_psprintf(ctx->pool,dcache->inv_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x*dcache->count_y)); + apr_psprintf(ctx->pool,cache->inv_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/cache->count_x*cache->count_y)); if(!*path) { ctx->set_error(ctx,500, "failed to allocate tile key"); } } #ifdef DEBUG -static void check_tiff_format(mapcache_context *ctx, mapcache_tile *tile, TIFF *hTIFF, const char *filename) +static void check_tiff_format(mapcache_context *ctx, mapcache_cache_tiff *cache, mapcache_tile *tile, TIFF *hTIFF, const char *filename) { - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)tile->tileset->cache; uint32 imwidth,imheight,tilewidth,tileheight; int16 planarconfig,orientation; uint16 compression; @@ -192,8 +190,8 @@ * configured for the cache */ level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); if( tilewidth != tile->grid_link->grid->tile_sx || tileheight != tile->grid_link->grid->tile_sy || imwidth != tile->grid_link->grid->tile_sx * ntilesx || @@ -213,13 +211,12 @@ } #endif -static int _mapcache_cache_tiff_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_tiff_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; TIFF *hTIFF; - mapcache_cache_tiff *dcache; - _mapcache_cache_tiff_tile_key(ctx, tile, &filename); - dcache = (mapcache_cache_tiff*)tile->tileset->cache; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; + _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } @@ -246,15 +243,15 @@ #ifdef DEBUG - check_tiff_format(ctx,tile,hTIFF,filename); + check_tiff_format(ctx,cache,tile,hTIFF,filename); if(GC_HAS_ERROR(ctx)) { MyTIFFClose(hTIFF); return MAPCACHE_FALSE; } #endif level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); /* x offset of the tile along a row */ tiff_offx = tile->x % ntilesx; @@ -286,7 +283,7 @@ return MAPCACHE_FALSE; } -static void _mapcache_cache_tiff_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_tiff_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { ctx->set_error(ctx,500,"TIFF cache tile deleting not implemented"); } @@ -299,14 +296,13 @@ * \private \memberof mapcache_cache_tiff * \sa mapcache_cache::tile_get() */ -static int _mapcache_cache_tiff_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_tiff_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; TIFF *hTIFF = NULL; int rv; - mapcache_cache_tiff *dcache; - _mapcache_cache_tiff_tile_key(ctx, tile, &filename); - dcache = (mapcache_cache_tiff*)tile->tileset->cache; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; + _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } @@ -347,7 +343,7 @@ #ifdef DEBUG - check_tiff_format(ctx,tile,hTIFF,filename); + check_tiff_format(ctx,cache,tile,hTIFF,filename); if(GC_HAS_ERROR(ctx)) { MyTIFFClose(hTIFF); return MAPCACHE_FAILURE; @@ -359,8 +355,8 @@ * file for lower zoom levels */ level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); /* x offset of the tile along a row */ tiff_offx = tile->x % ntilesx; @@ -441,7 +437,7 @@ memcpy(tile->encoded_data->buf,jpegtable_ptr,(jpegtable_size-2)); /* advance the data pointer to after the header data */ - bufptr = tile->encoded_data->buf + (jpegtable_size-2); + bufptr = ((char *)tile->encoded_data->buf) + (jpegtable_size-2); /* go to the specified offset in the tiff file, plus 2 bytes */ @@ -508,17 +504,16 @@ * \private \memberof mapcache_cache_tiff * \sa mapcache_cache::tile_set() */ -static void _mapcache_cache_tiff_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_tiff_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { #ifdef USE_TIFF_WRITE char *filename; TIFF *hTIFF = NULL; int rv; + void *lock; int create; - char errmsg[120]; - mapcache_cache_tiff *dcache; + mapcache_cache_tiff *cache; mapcache_image_format_jpeg *format; - char *hackptr1,*hackptr2; int tilew; int tileh; unsigned char *rgb; @@ -530,9 +525,9 @@ int tiff_offx, tiff_offy; /* the x and y offset of the tile inside the tiff image */ int tiff_off; /* the index of the tile inside the list of tiles of the tiff image */ - _mapcache_cache_tiff_tile_key(ctx, tile, &filename); - dcache = (mapcache_cache_tiff*)tile->tileset->cache; - format = (mapcache_image_format_jpeg*) dcache->format; + cache = (mapcache_cache_tiff*)pcache; + _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); + format = (mapcache_image_format_jpeg*) cache->format; if(GC_HAS_ERROR(ctx)) { return; } @@ -544,27 +539,8 @@ /* * create the directory where the tiff file will be stored */ - - /* find the location of the last '/' in the string */ - hackptr2 = hackptr1 = filename; - while(*hackptr1) { - if(*hackptr1 == '/') - hackptr2 = hackptr1; - hackptr1++; - } - *hackptr2 = '\0'; - - if(APR_SUCCESS != (rv = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - /* - * apr_dir_make_recursive sometimes sends back this error, although it should not. - * ignore this one - */ - if(!APR_STATUS_IS_EEXIST(rv)) { - ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(rv,errmsg,120)); - return; - } - } - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); tilew = tile->grid_link->grid->tile_sx; tileh = tile->grid_link->grid->tile_sy; @@ -592,7 +568,7 @@ * aquire a lock on the tiff file. */ - while(mapcache_lock_or_wait_for_resource(ctx,filename) == MAPCACHE_FALSE); + while(mapcache_lock_or_wait_for_resource(ctx,(cache->locker?cache->locker:ctx->config->locker),filename, &lock) == MAPCACHE_FALSE); /* check if the tiff file exists already */ rv = apr_stat(&finfo,filename,0,ctx->pool); @@ -615,11 +591,12 @@ * file for lower zoom levels */ level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); if(create) { #ifdef USE_GEOTIFF - double adfPixelScale[3], adfTiePoints[6], bbox[4]; + double adfPixelScale[3], adfTiePoints[6]; + mapcache_extent bbox; GTIF *gtif; int x,y; #endif @@ -679,16 +656,16 @@ /* top left tile x,y */ - x = (tile->x / dcache->count_x)*(dcache->count_x); - y = (tile->y / dcache->count_y)*(dcache->count_y) + ntilesy - 1; + x = (tile->x / cache->count_x)*(cache->count_x); + y = (tile->y / cache->count_y)*(cache->count_y) + ntilesy - 1; mapcache_grid_get_extent(ctx, tile->grid_link->grid, - x,y,tile->z,bbox); + x,y,tile->z,&bbox); adfTiePoints[0] = 0.0; adfTiePoints[1] = 0.0; adfTiePoints[2] = 0.0; - adfTiePoints[3] = bbox[0]; - adfTiePoints[4] = bbox[3]; + adfTiePoints[3] = bbox.minx; + adfTiePoints[4] = bbox.maxy; adfTiePoints[5] = 0.0; TIFFSetField( hTIFF, TIFFTAG_GEOTIEPOINTS, 6, adfTiePoints ); } @@ -737,7 +714,7 @@ close_tiff: if(hTIFF) MyTIFFClose(hTIFF); - mapcache_unlock_resource(ctx,filename); + mapcache_unlock_resource(ctx,cache->locker?cache->locker:ctx->config->locker,filename, lock); #else ctx->set_error(ctx,500,"tiff write support disabled by default"); #endif @@ -747,76 +724,73 @@ /** * \private \memberof mapcache_cache_tiff */ -static void _mapcache_cache_tiff_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_tiff_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) { ezxml_t cur_node; - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)cache; - ezxml_t xcount; - ezxml_t ycount; - ezxml_t xformat; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; char * format_name; mapcache_image_format *pformat; if ((cur_node = ezxml_child(node,"template")) != NULL) { char *fmt; - dcache->filename_template = apr_pstrdup(ctx->pool,cur_node->txt); + cache->filename_template = apr_pstrdup(ctx->pool,cur_node->txt); fmt = (char*)ezxml_attr(cur_node,"x_fmt"); if(fmt && *fmt) { - dcache->x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"y_fmt"); if(fmt && *fmt) { - dcache->y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->y_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"z_fmt"); if(fmt && *fmt) { - dcache->z_fmt = apr_pstrdup(ctx->pool,fmt); + cache->z_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_x_fmt"); if(fmt && *fmt) { - dcache->inv_x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_y_fmt"); if(fmt && *fmt) { - dcache->inv_y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_y_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"div_x_fmt"); if(fmt && *fmt) { - dcache->div_x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->div_x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"div_y_fmt"); if(fmt && *fmt) { - dcache->div_y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->div_y_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_div_x_fmt"); if(fmt && *fmt) { - dcache->inv_div_x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_div_x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_div_y_fmt"); if(fmt && *fmt) { - dcache->inv_div_y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_div_y_fmt = apr_pstrdup(ctx->pool,fmt); } } - xcount = ezxml_child(node,"xcount"); - if(xcount && xcount->txt && *xcount->txt) { + cur_node = ezxml_child(node,"xcount"); + if(cur_node && cur_node->txt && *cur_node->txt) { char *endptr; - dcache->count_x = (int)strtol(xcount->txt,&endptr,10); + cache->count_x = (int)strtol(cur_node->txt,&endptr,10); if(*endptr != 0) { - ctx->set_error(ctx,400,"failed to parse xcount value %s for tiff cache %s", xcount->txt,cache->name); + ctx->set_error(ctx,400,"failed to parse xcount value %s for tiff cache %s", cur_node->txt,pcache->name); return; } } - ycount = ezxml_child(node,"ycount"); - if(ycount && ycount->txt && *ycount->txt) { + cur_node = ezxml_child(node,"ycount"); + if(cur_node && cur_node->txt && *cur_node->txt) { char *endptr; - dcache->count_y = (int)strtol(ycount->txt,&endptr,10); + cache->count_y = (int)strtol(cur_node->txt,&endptr,10); if(*endptr != 0) { - ctx->set_error(ctx,400,"failed to parse ycount value %s for tiff cache %s", ycount->txt,cache->name); + ctx->set_error(ctx,400,"failed to parse ycount value %s for tiff cache %s", cur_node->txt,pcache->name); return; } } - xformat = ezxml_child(node,"format"); - if(xformat && xformat->txt && *xformat->txt) { - format_name = xformat->txt; + cur_node = ezxml_child(node,"format"); + if(cur_node && cur_node->txt && *cur_node->txt) { + format_name = cur_node->txt; } else { format_name = "JPEG"; } @@ -824,31 +798,37 @@ config,format_name); if(!pformat) { ctx->set_error(ctx,500,"TIFF cache %s references unknown image format %s", - cache->name, format_name); + pcache->name, format_name); return; } if(pformat->type != GC_JPEG) { ctx->set_error(ctx,500,"TIFF cache %s can only reference a JPEG image format", - cache->name); + pcache->name); return; } - dcache->format = (mapcache_image_format_jpeg*)pformat; + cache->format = (mapcache_image_format_jpeg*)pformat; + + cur_node = ezxml_child(node,"locker"); + if(cur_node) { + mapcache_config_parse_locker(ctx, cur_node, &cache->locker); + } + } /** * \private \memberof mapcache_cache_tiff */ -static void _mapcache_cache_tiff_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, +static void _mapcache_cache_tiff_configuration_post_config(mapcache_context *ctx, mapcache_cache *pcache, mapcache_cfg *cfg) { - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)cache; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; /* check all required parameters are configured */ - if((!dcache->filename_template || !strlen(dcache->filename_template))) { - ctx->set_error(ctx, 400, "tiff cache %s has no template pattern",dcache->cache.name); + if((!cache->filename_template || !strlen(cache->filename_template))) { + ctx->set_error(ctx, 400, "tiff cache %s has no template pattern",cache->cache.name); return; } - if(dcache->count_x <= 0 || dcache->count_y <= 0) { - ctx->set_error(ctx, 400, "tiff cache %s has invalid count (%d,%d)",dcache->count_x,dcache->count_y); + if(cache->count_x <= 0 || cache->count_y <= 0) { + ctx->set_error(ctx, 400, "tiff cache %s has invalid count (%d,%d)",cache->count_x,cache->count_y); return; } } diff -Nru mapcache-1.2.1/lib/configuration.c mapcache-1.4.1/lib/configuration.c --- mapcache-1.2.1/lib/configuration.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/configuration.c 2016-02-25 15:47:16.000000000 +0000 @@ -35,47 +35,12 @@ void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi) { - apr_dir_t *lockdir; - apr_status_t rv; - char errmsg[120]; char *url; mapcache_configuration_parse_xml(ctx,filename,config); - GC_CHECK_ERROR(ctx); - if(!config->lockdir || !strlen(config->lockdir)) { - config->lockdir = apr_pstrdup(ctx->pool, "/tmp"); - } - rv = apr_dir_open(&lockdir,config->lockdir,ctx->pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500, "failed to open lock directory %s: %s" - ,config->lockdir,apr_strerror(rv,errmsg,120)); - return; - } - - /* only remove lockfiles if we're not in cgi mode */ - if(!cgi) { - apr_finfo_t finfo; - while ((apr_dir_read(&finfo, APR_FINFO_DIRENT|APR_FINFO_TYPE|APR_FINFO_NAME, lockdir)) == APR_SUCCESS) { - if(finfo.filetype == APR_REG) { - if(!strncmp(finfo.name, MAPCACHE_LOCKFILE_PREFIX, strlen(MAPCACHE_LOCKFILE_PREFIX))) { - ctx->log(ctx,MAPCACHE_WARN,"found old lockfile %s/%s, deleting it",config->lockdir, - finfo.name); - rv = apr_file_remove(apr_psprintf(ctx->pool,"%s/%s",config->lockdir, finfo.name),ctx->pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500, "failed to remove lockfile %s: %s",finfo.name,apr_strerror(rv,errmsg,120)); - return; - } - - } - - } - } - } - apr_dir_close(lockdir); - /* if we were suppplied with an onlineresource, make sure it ends with a / */ if(NULL != (url = (char*)apr_table_get(config->metadata,"url"))) { char *urlend = url + strlen(url)-1; @@ -254,8 +219,6 @@ } mapcache_configuration_add_grid(cfg,grid,"g"); - /* default retry interval is 1/100th of a second, i.e. 10000 microseconds */ - cfg->lock_retry_interval = 10000; cfg->loglevel = MAPCACHE_WARN; cfg->autoreload = 0; diff -Nru mapcache-1.2.1/lib/configuration_xml.c mapcache-1.4.1/lib/configuration_xml.c --- mapcache-1.2.1/lib/configuration_xml.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/configuration_xml.c 2016-02-25 15:47:16.000000000 +0000 @@ -96,6 +96,7 @@ char *name = (char*)ezxml_attr(dimension_node,"name"); char *type = (char*)ezxml_attr(dimension_node,"type"); char *unit = (char*)ezxml_attr(dimension_node,"unit"); + char *skip_validation = (char*)ezxml_attr(dimension_node,"skip_validation"); char *default_value = (char*)ezxml_attr(dimension_node,"default"); mapcache_dimension *dimension = NULL; @@ -112,6 +113,8 @@ dimension = mapcache_dimension_regex_create(ctx->pool); } else if(!strcmp(type,"intervals")) { dimension = mapcache_dimension_intervals_create(ctx->pool); + } else if(!strcmp(type,"sqlite")) { + dimension = mapcache_dimension_sqlite_create(ctx->pool); } else if(!strcmp(type,"time")) { ctx->set_error(ctx,501,"time dimension type not implemented yet"); return; @@ -130,6 +133,10 @@ if(unit && *unit) { dimension->unit = apr_pstrdup(ctx->pool,unit); } + + if(skip_validation && !strcmp(skip_validation,"true")) { + dimension->skip_validation = MAPCACHE_TRUE; + } if(default_value && *default_value) { dimension->default_value = apr_pstrdup(ctx->pool,default_value); @@ -387,6 +394,8 @@ compression = MAPCACHE_COMPRESSION_FAST; } else if(!strcmp(cur_node->txt, "best")) { compression = MAPCACHE_COMPRESSION_BEST; + } else if(!strcmp(cur_node->txt, "none")) { + compression = MAPCACHE_COMPRESSION_DISABLE; } else { ctx->set_error(ctx, 400, "unknown compression type %s for format \"%s\"", cur_node->txt, name); return; @@ -496,6 +505,20 @@ } if(!strcmp(type,"disk")) { cache = mapcache_cache_disk_create(ctx); + } else if(!strcmp(type,"fallback")) { + cache = mapcache_cache_fallback_create(ctx); + } else if(!strcmp(type,"multitier")) { + cache = mapcache_cache_multitier_create(ctx); + } else if(!strcmp(type,"composite")) { + cache = mapcache_cache_composite_create(ctx); + } else if(!strcmp(type,"rest")) { + cache = mapcache_cache_rest_create(ctx); + } else if(!strcmp(type,"s3")) { + cache = mapcache_cache_s3_create(ctx); + } else if(!strcmp(type,"azure")) { + cache = mapcache_cache_azure_create(ctx); + } else if(!strcmp(type,"google")) { + cache = mapcache_cache_google_create(ctx); } else if(!strcmp(type,"bdb")) { #ifdef USE_BDB cache = mapcache_cache_bdb_create(ctx); @@ -538,6 +561,18 @@ ctx->set_error(ctx,400, "failed to add cache \"%s\": tiff support is not available on this build",name); return; #endif + } else if(!strcmp(type,"couchbase")) { +#ifdef USE_COUCHBASE + cache = mapcache_cache_couchbase_create(ctx); +#else + ctx->set_error(ctx, 400, "failed to add cache \"%s\": couchbase support is not available on this build", name); +#endif + } else if(!strcmp(type,"riak")) { +#ifdef USE_RIAK + cache = mapcache_cache_riak_create(ctx); +#else + ctx->set_error(ctx, 400, "failed to add cache \"%s\": riak support is not available on this build", name); +#endif } else { ctx->set_error(ctx, 400, "unknown cache type %s for cache \"%s\"", type, name); return; @@ -632,6 +667,7 @@ gridlink->maxz = grid->nlevels; gridlink->grid_limits = (mapcache_extent_i*)apr_pcalloc(ctx->pool,grid->nlevels*sizeof(mapcache_extent_i)); gridlink->outofzoom_strategy = MAPCACHE_OUTOFZOOM_NOTCONFIGURED; + gridlink->intermediate_grids = apr_array_make(ctx->pool,1,sizeof(mapcache_grid_link*)); restrictedExtent = (char*)ezxml_attr(cur_node,"restricted_extent"); if(restrictedExtent) { @@ -667,6 +703,11 @@ return; } } + sTolerance = (char*)ezxml_attr(cur_node,"use_wms_intermediate_resolutions"); + if(sTolerance && !strcmp(sTolerance,"true")) { + mapcache_grid_link *intermediate_gridlink = apr_pcalloc(ctx->pool,sizeof(mapcache_grid_link)); + APR_ARRAY_PUSH(gridlink->intermediate_grids,mapcache_grid_link*) = intermediate_gridlink; + } mapcache_grid_compute_limits(grid,extent,gridlink->grid_limits,tolerance); @@ -741,6 +782,40 @@ if(!havewgs84bbox && !strcasecmp(grid->srs,"EPSG:4326")) { tileset->wgs84bbox = *extent; } + + if(gridlink->intermediate_grids->nelts > 0) { + double factor = 0.5, unitheight,unitwidth; + int i; + mapcache_grid_link *igl = APR_ARRAY_IDX(gridlink->intermediate_grids, 0, mapcache_grid_link*); + igl->restricted_extent = gridlink->restricted_extent; + igl->minz = gridlink->minz; + igl->max_cached_zoom = gridlink->max_cached_zoom - 1; + igl->maxz = gridlink->maxz - 1; + igl->outofzoom_strategy = gridlink->outofzoom_strategy; + igl->grid = mapcache_grid_create(ctx->pool); + igl->grid->extent = gridlink->grid->extent; + igl->grid->name = apr_psprintf(ctx->pool,"%s_intermediate_%g",gridlink->grid->name,factor); + igl->grid->nlevels = gridlink->grid->nlevels - 1; + igl->grid->origin = gridlink->grid->origin; + igl->grid->srs = gridlink->grid->srs; + igl->grid->srs_aliases = gridlink->grid->srs_aliases; + igl->grid->unit = gridlink->grid->unit; + igl->grid->tile_sx = gridlink->grid->tile_sx + gridlink->grid->tile_sx * factor; + igl->grid->tile_sy = gridlink->grid->tile_sy + gridlink->grid->tile_sy * factor; + igl->grid->levels = (mapcache_grid_level**)apr_pcalloc(ctx->pool, igl->grid->nlevels*sizeof(mapcache_grid_level*)); + for(i=0; igrid->nlevels; i++) { + mapcache_grid_level *level = (mapcache_grid_level*)apr_pcalloc(ctx->pool,sizeof(mapcache_grid_level)); + level->resolution = gridlink->grid->levels[i]->resolution + (gridlink->grid->levels[i+1]->resolution - gridlink->grid->levels[i]->resolution) * factor; + unitheight = igl->grid->tile_sy * level->resolution; + unitwidth = igl->grid->tile_sx * level->resolution; + + level->maxy = ceil((igl->grid->extent.maxy-igl->grid->extent.miny - 0.01* unitheight)/unitheight); + level->maxx = ceil((igl->grid->extent.maxx-igl->grid->extent.minx - 0.01* unitwidth)/unitwidth); + igl->grid->levels[i] = level; + } + igl->grid_limits = (mapcache_extent_i*)apr_pcalloc(ctx->pool,igl->grid->nlevels*sizeof(mapcache_extent_i)); + mapcache_grid_compute_limits(igl->grid,extent,igl->grid_limits,tolerance); + } APR_ARRAY_PUSH(tileset->grid_links,mapcache_grid_link*) = gridlink; } @@ -762,7 +837,7 @@ " but it is not configured", name, cur_node->txt); return; } - tileset->cache = cache; + tileset->_cache = cache; } if ((cur_node = ezxml_child(node,"source")) != NULL) { @@ -910,6 +985,7 @@ } + void mapcache_configuration_parse_xml(mapcache_context *ctx, const char *filename, mapcache_cfg *config) { ezxml_t doc, node; @@ -1078,20 +1154,35 @@ } } - if((node = ezxml_child(doc,"lock_dir")) != NULL) { - config->lockdir = apr_pstrdup(ctx->pool, node->txt); + if((node = ezxml_child(doc,"locker")) != NULL) { + mapcache_config_parse_locker(ctx,node,&config->locker); + GC_CHECK_ERROR(ctx); } else { - config->lockdir = apr_pstrdup(ctx->pool,"/tmp"); - } + /* backwards compatibility */ + int micro_retry; + mapcache_locker_disk *ldisk; + config->locker = mapcache_locker_disk_create(ctx); + ldisk = (mapcache_locker_disk*)config->locker; + if((node = ezxml_child(doc,"lock_dir")) != NULL) { + ldisk->dir = apr_pstrdup(ctx->pool, node->txt); + } else { + ldisk->dir = apr_pstrdup(ctx->pool,"/tmp"); + } - if((node = ezxml_child(doc,"lock_retry")) != NULL) { - char *endptr; - config->lock_retry_interval = (unsigned int)strtol(node->txt,&endptr,10); - if(*endptr != 0 || config->lock_retry_interval < 0) { - ctx->set_error(ctx, 400, "failed to parse lock_retry microseconds \"%s\". Expecting a positive integer", - node->txt); - return; + if((node = ezxml_child(doc,"lock_retry")) != NULL) { + char *endptr; + micro_retry = strtol(node->txt,&endptr,10); + if(*endptr != 0 || micro_retry <= 0) { + ctx->set_error(ctx, 400, "failed to parse lock_retry microseconds \"%s\". Expecting a positive integer", + node->txt); + return; + } + } else { + /* default retry interval is 1/100th of a second, i.e. 10000 microseconds */ + micro_retry = 10000; } + config->locker->retry_interval = micro_retry / 1000000.0; + config->locker->timeout=120; } if((node = ezxml_child(doc,"threaded_fetching")) != NULL) { diff -Nru mapcache-1.2.1/lib/connection_pool.c mapcache-1.4.1/lib/connection_pool.c --- mapcache-1.2.1/lib/connection_pool.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/lib/connection_pool.c 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,191 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache connection pooling + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include +#include "mapcache.h" + +struct mapcache_connection_pool { + apr_pool_t *server_pool; + apr_reslist_t *connexions; +}; + + +struct mapcache_pooled_connection_container { + mapcache_pooled_connection *head; + apr_pool_t *pool; + unsigned int max_list_size; +}; + + + +struct mapcache_pooled_connection_private_data { + char *key; + mapcache_connection_destructor destructor; + mapcache_pooled_connection *next; + mapcache_pooled_connection_container *pcc; +}; + +static apr_status_t mapcache_connection_container_creator(void **conn_, void *params, apr_pool_t *pool) { + mapcache_pooled_connection_container *pcc; + pcc = calloc(1, sizeof(mapcache_pooled_connection_container)); + pcc->max_list_size = 10; + pcc->pool = pool; + *conn_ = pcc; + return APR_SUCCESS; +} + +static apr_status_t mapcache_connection_container_destructor(void *conn_, void *params, apr_pool_t *pool) { + mapcache_pooled_connection_container *pcc = (mapcache_pooled_connection_container*)conn_; + mapcache_pooled_connection *pc = pcc->head; + while(pc) { + mapcache_pooled_connection *this = pc; + this->private->destructor(this->connection); + free(this->private->key); + pc = this->private->next; + free(this); + } + free(pcc); + return MAPCACHE_SUCCESS; +} + + +apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool) { + apr_status_t rv; + *cp = apr_pcalloc(server_pool, sizeof(mapcache_connection_pool)); + (*cp)->server_pool = server_pool; + rv = apr_reslist_create(&((*cp)->connexions), 1, 5, 200, 60*1000000, + mapcache_connection_container_creator, + mapcache_connection_container_destructor, + NULL, + server_pool); + return rv; +} + +mapcache_pooled_connection* mapcache_connection_pool_get_connection(mapcache_context *ctx, char *key, + mapcache_connection_constructor constructor, mapcache_connection_destructor destructor, + void *params) { + apr_status_t rv; + int count = 0; + mapcache_pooled_connection_container *pcc; + mapcache_pooled_connection *pc,*pred=NULL; + rv = apr_reslist_acquire(ctx->connection_pool->connexions, (void**)&pcc); + if(rv != APR_SUCCESS || !pcc) { + char errmsg[120]; + ctx->set_error(ctx,500, "failed to acquire connection from mapcache connection pool: (%s)", apr_strerror(rv, errmsg,120)); + return NULL; + } + + /* loop through existing connections to see if we find one matching the given key */ + pc = pcc->head; + while(pc) { + count++; + if(!strcmp(key,pc->private->key)) { + /* move current connection to head of list, and return it. We only move the connection + to the front of the list if it wasn't in the first 2 connections, as in the seeding + case we are always alternating between read and write operations (i.e. potentially + 2 different connections and in that cas we end up switching connections each time + there's an access */ + if(pc != pcc->head && count>2) { + assert(pred); + pred->private->next = pc->private->next; + pc->private->next = pcc->head; + pcc->head = pc; + } + return pc; + } + pred = pc; + pc = pc->private->next; + } + + /* connection not found in pool */ + pc = calloc(1,sizeof(mapcache_pooled_connection)); + /* + ctx->log(ctx, MAPCACHE_DEBUG, "calling constructor for pooled connection (%s)", key); + */ + constructor(ctx, &pc->connection, params); + if(GC_HAS_ERROR(ctx)) { + free(pc); + apr_reslist_release(ctx->connection_pool->connexions, pcc); + return NULL; + } + + pc->private = calloc(1,sizeof(mapcache_pooled_connection_private_data)); + pc->private->key = strdup(key); + pc->private->destructor = destructor; + pc->private->next = pcc->head; + pc->private->pcc = pcc; + + if(count == pcc->max_list_size) { + /* max number of connections atained, we must destroy the last one that was used */ + mapcache_pooled_connection *opc; + opc = pcc->head; + count = 1; + while(count < pcc->max_list_size) { + pred = opc; + opc = opc->private->next; + count++; + } + ctx->log(ctx, MAPCACHE_DEBUG, "tearing down pooled connection (%s) to make room", opc->private->key); + opc->private->destructor(opc->connection); + free(opc->private->key); + free(opc->private); + free(opc); + if(pred) { + pred->private->next = NULL; + } + } + pcc->head = pc; + return pc; + +} +void mapcache_connection_pool_invalidate_connection(mapcache_context *ctx, mapcache_pooled_connection *connection) { + mapcache_pooled_connection_container *pcc = connection->private->pcc; + mapcache_pooled_connection *pc = pcc->head, *pred=NULL; + while(pc) { + if(pc == connection) { + if(pred) { + pred->private->next = pc->private->next; + } else { + pcc->head = pc->private->next; + } + pc->private->destructor(pc->connection); + free(pc->private->key); + free(pc); + } + pred = pc; + pc = pc->private->next; + } +} + +void mapcache_connection_pool_release_connection(mapcache_context *ctx, mapcache_pooled_connection *connection) { + if(connection) { + mapcache_pooled_connection_container *pcc = connection->private->pcc; + apr_reslist_release(ctx->connection_pool->connexions,(void*)pcc); + } +} + diff -Nru mapcache-1.2.1/lib/core.c mapcache-1.4.1/lib/core.c --- mapcache-1.2.1/lib/core.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/core.c 2016-02-25 15:47:16.000000000 +0000 @@ -196,10 +196,13 @@ { int expires = 0; mapcache_http_response *response; - int i,is_empty=1 /* response image is initially empty */; char *timestr; - mapcache_image *base=NULL; - mapcache_image_format *format = NULL; + mapcache_image *base; + mapcache_image_format *format; + mapcache_image_format_type t; + int i,is_empty=1; /* response image is initially empty */; + base=NULL; + format = NULL; #ifdef DEBUG if(req_tile->ntiles ==0) { @@ -209,11 +212,22 @@ #endif response = mapcache_http_response_create(ctx->pool); + if(ctx->supports_redirects && req_tile->ntiles == 1) { + req_tile->tiles[0]->allow_redirect = 1; + } + mapcache_prefetch_tiles(ctx,req_tile->tiles,req_tile->ntiles); if(GC_HAS_ERROR(ctx)) return NULL; + if(req_tile->tiles[0]->redirect) { + response->code = 302; + apr_table_set(response->headers,"Location",req_tile->tiles[0]->redirect); + response->data = mapcache_buffer_create(0, ctx->pool); + return response; + } + /* loop through tiles, and eventually merge them vertically together */ for(i=0; intiles; i++) { mapcache_tile *tile = req_tile->tiles[i]; /* shortcut */ @@ -222,7 +236,7 @@ if(tile->expires && (tile->expires < expires || expires == 0)) { expires = tile->expires; } - + if(tile->nodata) { /* treat the special case where the cache explicitely stated that the tile was empty, and we don't have any vertical merging to do */ @@ -233,8 +247,8 @@ } continue; } - - /* treat the most common case: + + /* treat the most common case: - we have a single tile request (i.e. isempty is true) - the cache returned the encoded image */ @@ -271,7 +285,7 @@ mapcache_image_merge(ctx, base, tile->raw_image); } else { /* we don't need to merge onto an existing tile and don't have access to the tile's encoded data. - * + * * we don't encode the tile's raw image data just yet because we might need to merge another one on top * of it later. */ @@ -283,8 +297,8 @@ if(!response->data) { /* we need to encode the raw image data*/ if(base) { - if(req_tile->format) { - format = req_tile->format; + if(req_tile->image_request.format) { + format = req_tile->image_request.format; } else { format = req_tile->tiles[0]->tileset->format; if(!format) { @@ -296,20 +310,20 @@ return NULL; } } else { + unsigned char empty[5] = {'#',0,0,0,0}; #ifdef DEBUG if(!is_empty) { ctx->set_error(ctx,500,"BUG: no image data to encode, but tile not marked as empty"); return NULL; } #endif - unsigned char empty[5] = {'#',0,0,0,0}; - response->data = mapcache_empty_png_decode(ctx,empty,&is_empty); /* is_empty is unchanged and left to 1 */ + response->data = mapcache_empty_png_decode(ctx,req_tile->tiles[0]->grid_link->grid->tile_sx, req_tile->tiles[0]->grid_link->grid->tile_sy, empty,&is_empty); /* is_empty is unchanged and left to 1 */ format = mapcache_configuration_get_image_format(ctx->config,"PNG8"); } } - + /* compute the content-type */ - mapcache_image_format_type t = mapcache_imageio_header_sniff(ctx,response->data); + t = mapcache_imageio_header_sniff(ctx,response->data); if(t == GC_PNG) apr_table_set(response->headers,"Content-Type","image/png"); else if(t == GC_JPEG) @@ -334,15 +348,18 @@ mapcache_tile ***maptiles; int *nmaptiles; mapcache_tile **tiles; + mapcache_grid_link **effectively_used_grid_links; mapcache_map *basemap = NULL; int ntiles = 0; int i; maptiles = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_tile**)); nmaptiles = apr_pcalloc(ctx->pool,nmaps*sizeof(int)); + effectively_used_grid_links = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_grid_link*)); for(i=0; itileset,maps[i]->grid_link, &maps[i]->extent, maps[i]->width, maps[i]->height, - &(nmaptiles[i]), &(maptiles[i])); + &(nmaptiles[i]), &(maptiles[i]), &(effectively_used_grid_links[i])); + if(GC_HAS_ERROR(ctx)) return NULL; ntiles += nmaptiles[i]; } tiles = apr_pcalloc(ctx->pool,ntiles * sizeof(mapcache_tile*)); @@ -379,7 +396,7 @@ } } if(hasdata) { - maps[i]->raw_image = mapcache_tileset_assemble_map_tiles(ctx,maps[i]->tileset,maps[i]->grid_link, + maps[i]->raw_image = mapcache_tileset_assemble_map_tiles(ctx,maps[i]->tileset,effectively_used_grid_links[i], &maps[i]->extent, maps[i]->width, maps[i]->height, nmaptiles[i], maptiles[i], mode); @@ -468,7 +485,7 @@ } if(basemap->raw_image) { - format = req_map->getmap_format; /* always defined, defaults to JPEG */ + format = req_map->image_request.format; /* always defined, defaults to JPEG */ response->data = format->write(ctx,basemap->raw_image,format); if(GC_HAS_ERROR(ctx)) { return NULL; @@ -516,16 +533,21 @@ mapcache_http *http; mapcache_http_response *response = mapcache_http_response_create(ctx->pool); response->data = mapcache_buffer_create(30000,ctx->pool); - http = req_proxy->http; + http = mapcache_http_clone(ctx, req_proxy->rule->http); if(req_proxy->pathinfo) { - http = mapcache_http_clone(ctx,http); if( (*(req_proxy->pathinfo)) == '/' || http->url[strlen(http->url)-1] == '/') http->url = apr_pstrcat(ctx->pool,http->url,req_proxy->pathinfo,NULL); else http->url = apr_pstrcat(ctx->pool,http->url,"/",req_proxy->pathinfo,NULL); } - mapcache_http_do_request_with_params(ctx,http,req_proxy->params,response->data,response->headers,&response->code); + http->url = mapcache_http_build_url(ctx,http->url,req_proxy->params); + http->post_body = req_proxy->post_buf; + http->post_len = req_proxy->post_len; + if(req_proxy->headers) { + apr_table_overlap(http->headers, req_proxy->headers, APR_OVERLAP_TABLES_SET); + } + mapcache_http_do_request(ctx,http, response->data,response->headers,&response->code); if(response->code !=0 && GC_HAS_ERROR(ctx)) { /* the http request was successful, but the server returned an error */ ctx->clear_errors(ctx); diff -Nru mapcache-1.2.1/lib/dimension.c mapcache-1.4.1/lib/dimension.c --- mapcache-1.2.1/lib/dimension.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/dimension.c 2016-02-25 15:47:16.000000000 +0000 @@ -35,14 +35,27 @@ #include #ifdef USE_SQLITE #include -#include -#include -#ifdef APR_HAS_THREADS -#include -#endif +#include #endif +#ifndef HAVE_TIMEGM +time_t timegm(struct tm *tm) +{ + time_t t, tdiff; + struct tm in, gtime, ltime; + + memcpy(&in, tm, sizeof(in)); + t = mktime(&in); + + memcpy(>ime, gmtime(&t), sizeof(gtime)); + memcpy(<ime, localtime(&t), sizeof(ltime)); + gtime.tm_isdst = ltime.tm_isdst; + tdiff = t - mktime(>ime); + memcpy(&in, tm, sizeof(in)); + return mktime(&in) + tdiff; +} +#endif static int _mapcache_dimension_intervals_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) { @@ -69,16 +82,15 @@ return MAPCACHE_FAILURE; } -static const char** _mapcache_dimension_intervals_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_intervals_print(mapcache_context *ctx, mapcache_dimension *dim) { mapcache_dimension_intervals *dimension = (mapcache_dimension_intervals*)dim; - const char **ret = (const char**)apr_pcalloc(ctx->pool,(dimension->nintervals+1)*sizeof(const char*)); + apr_array_header_t *ret = apr_array_make(ctx->pool,dimension->nintervals,sizeof(char*)); int i; for(i=0; inintervals; i++) { mapcache_interval *interval = &dimension->intervals[i]; - ret[i] = apr_psprintf(ctx->pool,"%g/%g/%g",interval->start,interval->end,interval->resolution); + APR_ARRAY_PUSH(ret,char*) = apr_psprintf(ctx->pool,"%g/%g/%g",interval->start,interval->end,interval->resolution); } - ret[i]=NULL; return ret; } @@ -149,12 +161,11 @@ return MAPCACHE_FAILURE; } -static const char** _mapcache_dimension_regex_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_regex_print(mapcache_context *ctx, mapcache_dimension *dim) { mapcache_dimension_regex *dimension = (mapcache_dimension_regex*)dim; - const char **ret = (const char**)apr_pcalloc(ctx->pool,2*sizeof(const char*)); - ret[0]=dimension->regex_string; - ret[1]=NULL; + apr_array_header_t *ret = apr_array_make(ctx->pool,1,sizeof(char*)); + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,dimension->regex_string); return ret; } @@ -212,15 +223,14 @@ return MAPCACHE_FAILURE; } -static const char** _mapcache_dimension_values_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_values_print(mapcache_context *ctx, mapcache_dimension *dim) { mapcache_dimension_values *dimension = (mapcache_dimension_values*)dim; - const char **ret = (const char**)apr_pcalloc(ctx->pool,(dimension->nvalues+1)*sizeof(const char*)); + apr_array_header_t *ret = apr_array_make(ctx->pool,dimension->nvalues,sizeof(char*)); int i; for(i=0; invalues; i++) { - ret[i] = dimension->values[i]; + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,dimension->values[i]); } - ret[i]=NULL; return ret; } @@ -308,105 +318,202 @@ #ifdef USE_SQLITE -static apr_hash_t *time_connection_pools = NULL; - -struct sqlite_time_conn { +struct sqlite_dimension_conn { sqlite3 *handle; - sqlite3_stmt *prepared_statement; - char *errmsg; + sqlite3_stmt **prepared_statements; + int n_statements; }; -static apr_status_t _sqlite_time_reslist_get_ro_connection(void **conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_dimension_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; - int flags; - mapcache_timedimension_sqlite *dim = (mapcache_timedimension_sqlite*) params; - struct sqlite_time_conn *conn = apr_pcalloc(pool, sizeof (struct sqlite_time_conn)); + int flags; + char *dbfile = (char*) params; + struct sqlite_dimension_conn *conn = calloc(1, sizeof (struct sqlite_dimension_conn)); *conn_ = conn; flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; - ret = sqlite3_open_v2(dim->dbfile, &conn->handle, flags, NULL); - + ret = sqlite3_open_v2(dbfile, &conn->handle, flags, NULL); + if (ret != SQLITE_OK) { - return APR_EGENERAL; + ctx->set_error(ctx,500,"failed to open sqlite dimension dbfile (%s): %s",dbfile,sqlite3_errmsg(conn->handle)); + sqlite3_close(conn->handle); + *conn_=NULL; + return; } sqlite3_busy_timeout(conn->handle, 300000); - return APR_SUCCESS; } -static apr_status_t _sqlite_time_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_dimension_connection_destructor(void *conn_) { - struct sqlite_time_conn *conn = (struct sqlite_time_conn*) conn_; - if(conn->prepared_statement) { - sqlite3_finalize(conn->prepared_statement); + struct sqlite_dimension_conn *conn = (struct sqlite_dimension_conn*) conn_; + while(conn->n_statements) { + conn->n_statements--; + if(conn->prepared_statements[conn->n_statements]) { + sqlite3_finalize(conn->prepared_statements[conn->n_statements]); + } } + free(conn->prepared_statements); sqlite3_close(conn->handle); - return APR_SUCCESS; + free(conn); } -static struct sqlite_time_conn* _sqlite_time_get_conn(mapcache_context *ctx, mapcache_timedimension_sqlite *dim) { - apr_status_t rv; - struct sqlite_time_conn *conn = NULL; - apr_reslist_t *pool = NULL; - if(!time_connection_pools || NULL == (pool = apr_hash_get(time_connection_pools,dim->timedimension.key, APR_HASH_KEY_STRING)) ) { - -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - - if(!time_connection_pools) { - time_connection_pools = apr_hash_make(ctx->process_pool); - } - - /* probably doesn't exist, unless the previous mutex locked us, so we check */ - pool = apr_hash_get(time_connection_pools,dim->timedimension.key, APR_HASH_KEY_STRING); - if(!pool) { - /* there where no existing connection pools, create them*/ - rv = apr_reslist_create(&pool, - 0 /* min */, - 10 /* soft max */, - 200 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _sqlite_time_reslist_get_ro_connection, /* resource constructor */ - _sqlite_time_reslist_free_connection, /* resource destructor */ - dim, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create sqlite time connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; +static mapcache_pooled_connection* _sqlite_time_dimension_get_conn(mapcache_context *ctx, mapcache_timedimension_sqlite *dim) { + mapcache_pooled_connection *pc = mapcache_connection_pool_get_connection(ctx,dim->timedimension.key, + mapcache_sqlite_dimension_connection_constructor, + mapcache_sqlite_dimension_connection_destructor, dim->dbfile); + return pc; +} +static mapcache_pooled_connection* _sqlite_dimension_get_conn(mapcache_context *ctx, mapcache_dimension_sqlite *dim) { + mapcache_pooled_connection *pc = mapcache_connection_pool_get_connection(ctx,dim->dimension.name, + mapcache_sqlite_dimension_connection_constructor, + mapcache_sqlite_dimension_connection_destructor, dim->dbfile); + return pc; +} + +static void _sqlite_dimension_release_conn(mapcache_context *ctx, mapcache_pooled_connection *pc) +{ + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_connection_pool_release_connection(ctx,pc); + } +} + + +static int _mapcache_dimension_sqlite_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +{ + mapcache_dimension_sqlite *dimension = (mapcache_dimension_sqlite*)dim; + struct sqlite_dimension_conn *conn = NULL; + int sqliteret,paramidx,ret=MAPCACHE_FAILURE; + mapcache_pooled_connection *pc; + pc = _sqlite_dimension_get_conn(ctx,dimension); + if (GC_HAS_ERROR(ctx)) { + return ret; + } + conn = pc->connection; + if(!conn->prepared_statements) { + conn->prepared_statements = calloc(2,sizeof(sqlite3_stmt*)); + conn->n_statements = 2; + } + if(!conn->prepared_statements[0]) { + sqliteret = sqlite3_prepare_v2(conn->handle, dimension->validate_query, -1, &conn->prepared_statements[0], NULL); + if(sqliteret != SQLITE_OK) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); + goto cleanup; + } + } + + paramidx = sqlite3_bind_parameter_index(conn->prepared_statements[0], ":dim"); + if (paramidx) { + sqliteret = sqlite3_bind_text(conn->prepared_statements[0], paramidx, *value, -1, SQLITE_STATIC); + if(sqliteret != SQLITE_OK) { + ctx->set_error(ctx,400, "sqlite dimension failed to bind :dim : %s", sqlite3_errmsg(conn->handle)); + goto cleanup; + } + } + do { + sqliteret = sqlite3_step(conn->prepared_statements[0]); + if (sqliteret != SQLITE_DONE && sqliteret != SQLITE_ROW && sqliteret != SQLITE_BUSY && sqliteret != SQLITE_LOCKED) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on query : %s (%d)", sqlite3_errmsg(conn->handle), sqliteret); + goto cleanup; + } + if(sqliteret == SQLITE_ROW) { + const char* dim_modified = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); + if(strcmp(dim_modified, *value)) { + *value = apr_pstrdup(ctx->pool, dim_modified); } - apr_hash_set(time_connection_pools,dim->timedimension.key,APR_HASH_KEY_STRING,pool); + ret = MAPCACHE_SUCCESS; } -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - pool = apr_hash_get(time_connection_pools,dim->timedimension.key, APR_HASH_KEY_STRING); - assert(pool); + } while (sqliteret == SQLITE_ROW || sqliteret == SQLITE_BUSY || sqliteret == SQLITE_LOCKED); + +cleanup: + if(conn->prepared_statements[0]) { + sqlite3_reset(conn->prepared_statements[0]); } - rv = apr_reslist_acquire(pool, (void **) &conn); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to aquire connection to time dimension sqlite backend: %s", (conn && conn->errmsg)?conn->errmsg:"unknown error"); - return NULL; + _sqlite_dimension_release_conn(ctx,pc); + + return ret; +} + +static apr_array_header_t* _mapcache_dimension_sqlite_print(mapcache_context *ctx, mapcache_dimension *dim) +{ + mapcache_dimension_sqlite *dimension = (mapcache_dimension_sqlite*)dim; + struct sqlite_dimension_conn *conn = NULL; + int sqliteret; + apr_array_header_t *ret = apr_array_make(ctx->pool,0,sizeof(char*)); + mapcache_pooled_connection *pc; + pc = _sqlite_dimension_get_conn(ctx,dimension); + if (GC_HAS_ERROR(ctx)) { + return ret; + } + conn = pc->connection; + if(!conn->prepared_statements) { + conn->prepared_statements = calloc(2,sizeof(sqlite3_stmt*)); + conn->n_statements = 2; } - return conn; + + if(!conn->prepared_statements[1]) { + sqliteret = sqlite3_prepare_v2(conn->handle, dimension->list_query, -1, &conn->prepared_statements[1], NULL); + if(sqliteret != SQLITE_OK) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); + goto cleanup; + } + } + do { + sqliteret = sqlite3_step(conn->prepared_statements[1]); + if (sqliteret != SQLITE_DONE && sqliteret != SQLITE_ROW && sqliteret != SQLITE_BUSY && sqliteret != SQLITE_LOCKED) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on query : %s (%d)", sqlite3_errmsg(conn->handle), sqliteret); + goto cleanup; + } + if(sqliteret == SQLITE_ROW) { + const char* sqdim = (const char*) sqlite3_column_text(conn->prepared_statements[1], 0); + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,sqdim); + } + } while (sqliteret == SQLITE_ROW || sqliteret == SQLITE_BUSY || sqliteret == SQLITE_LOCKED); + +cleanup: + if(conn->prepared_statements[1]) { + sqlite3_reset(conn->prepared_statements[1]); + } + _sqlite_dimension_release_conn(ctx,pc); + + return ret; } -static void _sqlite_time_release_conn(mapcache_context *ctx, mapcache_timedimension_sqlite *sdim, struct sqlite_time_conn *conn) + +static void _mapcache_dimension_sqlite_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, + ezxml_t node) { - apr_reslist_t *pool; - pool = apr_hash_get(time_connection_pools,sdim->timedimension.key, APR_HASH_KEY_STRING); + mapcache_dimension_sqlite *dimension; + ezxml_t child; - if (GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(pool, (void*) conn); + dimension = (mapcache_dimension_sqlite*)dim; + + child = ezxml_child(node,"dbfile"); + if(child) { + dimension->dbfile = apr_pstrdup(ctx->pool, child->txt); + } else { + ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); + return; + } + child = ezxml_child(node,"validate_query"); + if(child) { + dimension->validate_query = apr_pstrdup(ctx->pool, child->txt); + } else { + ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); + return; + } + child = ezxml_child(node,"list_query"); + if(child) { + dimension->list_query = apr_pstrdup(ctx->pool, child->txt); } else { - apr_reslist_release(pool, (void*) conn); + ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); + return; } + } + static void _bind_sqlite_timedimension_params(mapcache_context *ctx, sqlite3_stmt *stmt, sqlite3 *handle, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end) @@ -432,38 +539,36 @@ } } - if(extent) { - paramidx = sqlite3_bind_parameter_index(stmt, ":minx"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->minx); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :minx %s", sqlite3_errmsg(handle)); - return; - } + paramidx = sqlite3_bind_parameter_index(stmt, ":minx"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->minx:-DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :minx %s", sqlite3_errmsg(handle)); + return; } - paramidx = sqlite3_bind_parameter_index(stmt, ":miny"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->miny); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :miny %s", sqlite3_errmsg(handle)); - return; - } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":miny"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->miny:-DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :miny %s", sqlite3_errmsg(handle)); + return; } - paramidx = sqlite3_bind_parameter_index(stmt, ":maxx"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->maxx); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :maxx %s", sqlite3_errmsg(handle)); - return; - } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":maxx"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxx:DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :maxx %s", sqlite3_errmsg(handle)); + return; } - paramidx = sqlite3_bind_parameter_index(stmt, ":maxy"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->maxy); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :maxy %s", sqlite3_errmsg(handle)); - return; - } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":maxy"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxy:DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :maxy %s", sqlite3_errmsg(handle)); + return; } } @@ -490,47 +595,48 @@ mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end) { mapcache_timedimension_sqlite *sdim = (mapcache_timedimension_sqlite*)dim; int ret; - sqlite3_stmt *stmt; apr_array_header_t *time_ids = NULL; - struct sqlite_time_conn *conn = _sqlite_time_get_conn(ctx, sdim); + mapcache_pooled_connection *pc; + struct sqlite_dimension_conn *conn; + pc = _sqlite_time_dimension_get_conn(ctx,sdim); if (GC_HAS_ERROR(ctx)) { - if(conn) _sqlite_time_release_conn(ctx, sdim, conn); return NULL; } - stmt = conn->prepared_statement; - if(!stmt) { - ret = sqlite3_prepare_v2(conn->handle, sdim->query, -1, &conn->prepared_statement, NULL); + conn = pc->connection; + if(!conn->prepared_statements) { + conn->prepared_statements = calloc(1,sizeof(sqlite3_stmt*)); + conn->n_statements = 1; + } + if(!conn->prepared_statements[0]) { + ret = sqlite3_prepare_v2(conn->handle, sdim->query, -1, &conn->prepared_statements[0], NULL); if(ret != SQLITE_OK) { ctx->set_error(ctx, 500, "time sqlite backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); - _sqlite_time_release_conn(ctx, sdim, conn); + _sqlite_dimension_release_conn(ctx, pc); return NULL; } - stmt = conn->prepared_statement; } - - _bind_sqlite_timedimension_params(ctx,stmt,conn->handle,tileset,grid,extent,start,end); + + _bind_sqlite_timedimension_params(ctx,conn->prepared_statements[0],conn->handle,tileset,grid,extent,start,end); if(GC_HAS_ERROR(ctx)) { - sqlite3_reset(stmt); - _sqlite_time_release_conn(ctx, sdim, conn); + _sqlite_dimension_release_conn(ctx, pc); return NULL; } - + time_ids = apr_array_make(ctx->pool,0,sizeof(char*)); do { - ret = sqlite3_step(stmt); + ret = sqlite3_step(conn->prepared_statements[0]); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { ctx->set_error(ctx, 500, "sqlite backend failed on timedimension query : %s (%d)", sqlite3_errmsg(conn->handle), ret); - sqlite3_reset(stmt); - _sqlite_time_release_conn(ctx, sdim, conn); + _sqlite_dimension_release_conn(ctx, pc); return NULL; } if(ret == SQLITE_ROW) { - const char* time_id = (const char*) sqlite3_column_text(stmt, 0); + const char* time_id = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); APR_ARRAY_PUSH(time_ids,char*) = apr_pstrdup(ctx->pool,time_id); } } while (ret == SQLITE_ROW || ret == SQLITE_BUSY || ret == SQLITE_LOCKED); - sqlite3_reset(stmt); - _sqlite_time_release_conn(ctx, sdim, conn); + sqlite3_reset(conn->prepared_statements[0]); + _sqlite_dimension_release_conn(ctx, pc); return time_ids; } @@ -554,7 +660,7 @@ void _mapcache_timedimension_sqlite_parse_xml(mapcache_context *ctx, mapcache_timedimension *dim, ezxml_t node) { mapcache_timedimension_sqlite *sdim = (mapcache_timedimension_sqlite*)dim; ezxml_t child; - + child = ezxml_child(node,"dbfile"); if(child && child->txt && *child->txt) { sdim->dbfile = apr_pstrdup(ctx->pool,child->txt); @@ -573,8 +679,8 @@ #endif char *mapcache_ogc_strptime(const char *value, struct tm *ts, mapcache_time_interval_t *ti) { - memset (ts, '\0', sizeof (*ts)); char *valueptr; + memset (ts, '\0', sizeof (*ts)); valueptr = strptime(value,"%Y-%m-%dT%H:%M:%SZ",ts); *ti = MAPCACHE_TINTERVAL_SECOND; if(valueptr) return valueptr; @@ -611,10 +717,15 @@ ctx->set_error(ctx,400,"failed to parse time %s",value); return NULL; } - - if(*valueptr == '/') { + + if(*valueptr == '/' || (*valueptr == '-' && *(valueptr+1) == '-')) { /* we have a second (end) time */ - valueptr++; + if (*valueptr == '/') { + valueptr++; + } + else { + valueptr += 2; + } valueptr = mapcache_ogc_strptime(valueptr,&tm_end,&tie); if(!valueptr) { ctx->set_error(ctx,400,"failed to parse end time in %s",value); @@ -657,6 +768,22 @@ /* end loop */ } +mapcache_dimension* mapcache_dimension_sqlite_create(apr_pool_t *pool) +{ +#ifdef USE_SQLITE + mapcache_dimension_sqlite *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_sqlite)); + dimension->dimension.type = MAPCACHE_DIMENSION_SQLITE; + dimension->dbfile = NULL; + dimension->dimension.validate = _mapcache_dimension_sqlite_validate; + dimension->dimension.configuration_parse_xml = _mapcache_dimension_sqlite_parse_xml; + dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_sqlite_print; + return (mapcache_dimension*)dimension; +#else + return NULL; +#endif +} + + #ifdef USE_SQLITE mapcache_timedimension* mapcache_timedimension_sqlite_create(apr_pool_t *pool) { mapcache_timedimension_sqlite *dim = apr_pcalloc(pool, sizeof(mapcache_timedimension_sqlite)); diff -Nru mapcache-1.2.1/lib/ezxml.c mapcache-1.4.1/lib/ezxml.c --- mapcache-1.2.1/lib/ezxml.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/ezxml.c 2016-02-25 15:47:16.000000000 +0000 @@ -370,7 +370,7 @@ else *s = '\0'; // null terminate tag name for (i = 0; root->attr[i] && strcmp(n, root->attr[i][0]); i++); - while (*(n = ++s + strspn(s, EZXML_WS)) && *n != '>') { + while (*(n = s + 1 + strspn(s + 1, EZXML_WS)) && *n != '>') { if (*(s = n + strcspn(n, EZXML_WS))) *s = '\0'; // attr name else { ezxml_err(root, t, "malformed grid->levels[grid_link->minz]->resolution - resolution); - int i; + int i,g; + mapcache_grid_link *ret = grid_link; *level = 0; for(i=grid_link->minz + 1; imaxz; i++) { @@ -173,6 +174,20 @@ *level = i; } } + if(grid_link->intermediate_grids) { + for(g=0; gintermediate_grids->nelts; g++) { + mapcache_grid_link *igl = APR_ARRAY_IDX(grid_link->intermediate_grids, g, mapcache_grid_link*); + for(i=igl->minz; imaxz; i++) { + double curdst = fabs(igl->grid->levels[i]->resolution - resolution); + if(curdst + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "mapcache.h" +#include +#include + +#define SHA256_DIGEST_SIZE ( 256 / 8) +#define SHA256_BLOCK_SIZE ( 512 / 8) + +typedef unsigned char uint8; +typedef unsigned int uint32; +typedef unsigned long long uint64; + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA256_BLOCK_SIZE]; + uint32 h[8]; +} sha256_ctx; + +typedef struct { + sha256_ctx ctx_inside; + sha256_ctx ctx_outside; + + /* for hmac_reinit */ + sha256_ctx ctx_inside_reinit; + sha256_ctx ctx_outside_reinit; + + unsigned char block_ipad[SHA256_BLOCK_SIZE]; + unsigned char block_opad[SHA256_BLOCK_SIZE]; +} hmac_sha256_ctx; + + +#define SHFR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) +#define CH(x, y, z) ((x & y) ^ (~x & z)) +#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) + +#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3)) +#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10)) + +#define UNPACK32(x, str) \ +{ \ + *((str) + 3) = (uint8) ((x) ); \ + *((str) + 2) = (uint8) ((x) >> 8); \ + *((str) + 1) = (uint8) ((x) >> 16); \ + *((str) + 0) = (uint8) ((x) >> 24); \ +} + +#define PACK32(str, x) \ +{ \ + *(x) = ((uint32) *((str) + 3) ) \ + | ((uint32) *((str) + 2) << 8) \ + | ((uint32) *((str) + 1) << 16) \ + | ((uint32) *((str) + 0) << 24); \ +} + +#define UNPACK64(x, str) \ +{ \ + *((str) + 7) = (uint8) ((x) ); \ + *((str) + 6) = (uint8) ((x) >> 8); \ + *((str) + 5) = (uint8) ((x) >> 16); \ + *((str) + 4) = (uint8) ((x) >> 24); \ + *((str) + 3) = (uint8) ((x) >> 32); \ + *((str) + 2) = (uint8) ((x) >> 40); \ + *((str) + 1) = (uint8) ((x) >> 48); \ + *((str) + 0) = (uint8) ((x) >> 56); \ +} + +#define PACK64(str, x) \ +{ \ + *(x) = ((uint64) *((str) + 7) ) \ + | ((uint64) *((str) + 6) << 8) \ + | ((uint64) *((str) + 5) << 16) \ + | ((uint64) *((str) + 4) << 24) \ + | ((uint64) *((str) + 3) << 32) \ + | ((uint64) *((str) + 2) << 40) \ + | ((uint64) *((str) + 1) << 48) \ + | ((uint64) *((str) + 0) << 56); \ +} + +#define SHA256_SCR(i) \ +{ \ + w[i] = SHA256_F4(w[i - 2]) + w[i - 7] \ + + SHA256_F3(w[i - 15]) + w[i - 16]; \ +} + +uint32 sha256_h0[8] = + {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + +uint32 sha256_k[64] = + {0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +void sha256_transf(sha256_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint32 w[64]; + uint32 wv[8]; + uint32 t1, t2; + const unsigned char *sub_block; + int i,j; + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 6); + + for (j = 0; j < 16; j++) { + PACK32(&sub_block[j << 2], &w[j]); + } + + for (j = 16; j < 64; j++) { + SHA256_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } + } +} + + +void sha256_init(sha256_ctx *ctx) +{ + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha256_h0[i]; + } + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA256_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA256_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA256_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA256_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha256_final(sha256_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + int i; + + block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) + < (ctx->len % SHA256_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + + for (i = 0 ; i < 8; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +} + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest) +{ + sha256_ctx ctx; + + sha256_init(&ctx); + sha256_update(&ctx, message, len); + sha256_final(&ctx, digest); +} + +void hmac_sha256_init(hmac_sha256_ctx *ctx, const unsigned char *key, + unsigned int key_size) +{ + unsigned int fill; + unsigned int num; + + const unsigned char *key_used; + unsigned char key_temp[SHA256_DIGEST_SIZE]; + int i; + + if (key_size == SHA256_BLOCK_SIZE) { + key_used = key; + num = SHA256_BLOCK_SIZE; + } else { + if (key_size > SHA256_BLOCK_SIZE){ + num = SHA256_DIGEST_SIZE; + sha256(key, key_size, key_temp); + key_used = key_temp; + } else { /* key_size > SHA256_BLOCK_SIZE */ + key_used = key; + num = key_size; + } + fill = SHA256_BLOCK_SIZE - num; + + memset(ctx->block_ipad + num, 0x36, fill); + memset(ctx->block_opad + num, 0x5c, fill); + } + + for (i = 0; i < (int) num; i++) { + ctx->block_ipad[i] = key_used[i] ^ 0x36; + ctx->block_opad[i] = key_used[i] ^ 0x5c; + } + + sha256_init(&ctx->ctx_inside); + sha256_update(&ctx->ctx_inside, ctx->block_ipad, SHA256_BLOCK_SIZE); + + sha256_init(&ctx->ctx_outside); + sha256_update(&ctx->ctx_outside, ctx->block_opad, + SHA256_BLOCK_SIZE); + + /* for hmac_reinit */ + memcpy(&ctx->ctx_inside_reinit, &ctx->ctx_inside, + sizeof(sha256_ctx)); + memcpy(&ctx->ctx_outside_reinit, &ctx->ctx_outside, + sizeof(sha256_ctx)); +} + +void hmac_sha256_reinit(hmac_sha256_ctx *ctx) +{ + memcpy(&ctx->ctx_inside, &ctx->ctx_inside_reinit, + sizeof(sha256_ctx)); + memcpy(&ctx->ctx_outside, &ctx->ctx_outside_reinit, + sizeof(sha256_ctx)); +} + +void hmac_sha256_update(hmac_sha256_ctx *ctx, const unsigned char *message, + unsigned int message_len) +{ + sha256_update(&ctx->ctx_inside, message, message_len); +} + +void hmac_sha256_final(hmac_sha256_ctx *ctx, unsigned char *mac, + unsigned int mac_size) +{ + unsigned char digest_inside[SHA256_DIGEST_SIZE]; + unsigned char mac_temp[SHA256_DIGEST_SIZE]; + + sha256_final(&ctx->ctx_inside, digest_inside); + sha256_update(&ctx->ctx_outside, digest_inside, SHA256_DIGEST_SIZE); + sha256_final(&ctx->ctx_outside, mac_temp); + memcpy(mac, mac_temp, mac_size); +} + +void hmac_sha256(const unsigned char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + unsigned char *mac, unsigned mac_size) +{ + hmac_sha256_ctx ctx; + + hmac_sha256_init(&ctx, key, key_size); + hmac_sha256_update(&ctx, message, message_len); + hmac_sha256_final(&ctx, mac, mac_size); +} + +void sha_hex_encode(unsigned char *sha, unsigned int sha_size) { + int i = sha_size; + while(i--) { + char hex[3]; + sprintf(hex, "%02x", sha[i]); + memcpy(sha+2*i,hex,2); + } +} + + +#define HMAC_IPAD 0x36 +#define HMAC_OPAD 0x5c +#define HMAC_BLOCKSIZE 64 +#include + +static void mxor(void *dst, const void *src, size_t len) { + char const *s = src; + char *d = dst; + for (; len > 0; len--) + *d++ ^= *s++; + return; +} + +void hmac_sha1(const char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + void *mac) { + + apr_sha1_ctx_t inner; + apr_sha1_ctx_t outer; + unsigned char keypad[HMAC_BLOCKSIZE]; + unsigned char inner_digest[APR_SHA1_DIGESTSIZE]; + unsigned char long_digest[APR_SHA1_DIGESTSIZE]; + + /* Shorten the key down to the blocksize, anything more is useless */ + if (key_size > HMAC_BLOCKSIZE) { + apr_sha1_ctx_t context; + apr_sha1_init(&context); + apr_sha1_update_binary(&context, key, key_size); + apr_sha1_final(long_digest, &context); + key = long_digest; + key_size = APR_SHA1_DIGESTSIZE; + } + + /* Prepare and mask the inner portion of the key */ + memset(keypad, HMAC_IPAD, HMAC_BLOCKSIZE); + mxor(keypad, key, key_size); + + /* Compute the inner hash */ + apr_sha1_init(&inner); + apr_sha1_update_binary(&inner, keypad, HMAC_BLOCKSIZE); + apr_sha1_update(&inner, message, message_len); + apr_sha1_final(inner_digest, &inner); + + /* Prepare and mask the outer portion of the key */ + memset(keypad, HMAC_OPAD, HMAC_BLOCKSIZE); + mxor(keypad, key, key_size); + + /* Compute the outer hash */ + apr_sha1_init(&outer); + apr_sha1_update_binary(&outer, keypad, HMAC_BLOCKSIZE); + apr_sha1_update_binary(&outer, inner_digest, APR_SHA1_DIGESTSIZE); + apr_sha1_final(mac, &outer); +} + diff -Nru mapcache-1.2.1/lib/http.c mapcache-1.4.1/lib/http.c --- mapcache-1.2.1/lib/http.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/http.c 2016-02-25 15:47:16.000000000 +0000 @@ -73,12 +73,39 @@ return size*nmemb; } +/*update val replacing {foo_header} with the Value of foo_header from the orginal request */ +static void _header_replace_str(mapcache_context *ctx, apr_table_t *headers, char **val) { + char *value = *val; + char *start_tag, *end_tag; + size_t start_tag_offset; + start_tag = strchr(value,'{'); + while(start_tag) { + start_tag_offset = start_tag - value; /*record where we found the '{' so we can look for the next one after that spot + (avoids infinite loop if tag was not found/replaced) */ + *start_tag=0; + end_tag = strchr(start_tag+1,'}'); + if(end_tag) { + const char *header_value; + *end_tag=0; + header_value = apr_table_get(headers,start_tag+1); + if(header_value) { + value = apr_pstrcat(ctx->pool,value,header_value,end_tag+1,NULL); + } + *end_tag='}'; + } + *start_tag='{'; + start_tag = strchr(value+start_tag_offset+1,'{'); + } + *val = value; +} + void mapcache_http_do_request(mapcache_context *ctx, mapcache_http *req, mapcache_buffer *data, apr_table_t *headers, long *http_code) { CURL *curl_handle; char error_msg[CURL_ERROR_SIZE]; int ret; struct curl_slist *curl_headers=NULL; + struct _header_struct h; curl_handle = curl_easy_init(); @@ -95,7 +122,6 @@ if(headers != NULL) { /* intercept headers */ - struct _header_struct h; h.headers = headers; h.ctx=ctx; curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, _mapcache_curl_header_callback); @@ -115,13 +141,21 @@ apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; int i; for (i = 0; i < array->nelts; i++) { - curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",elts[i].val,NULL)); + char *val = elts[i].val; + if(val && strchr(val,'{') && ctx->headers_in) { + _header_replace_str(ctx,ctx->headers_in,&val); + } + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",val,NULL)); } } if(!req->headers || !apr_table_get(req->headers,"User-Agent")) { curl_headers = curl_slist_append(curl_headers, "User-Agent: "MAPCACHE_USERAGENT); } curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, curl_headers); + + if(req->post_body && req->post_len>0) { + curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, req->post_body); + } /* get it! */ ret = curl_easy_perform(curl_handle); if(http_code) @@ -144,16 +178,31 @@ mapcache_http_do_request(ctx,request,data,headers, http_code); } -/* calculate the length of the string formed by key=value&, and add it to cnt */ -#ifdef _WIN32 -static int _mapcache_key_value_strlen_callback(void *cnt, const char *key, const char *value) -{ -#else -static APR_DECLARE_NONSTD(int) _mapcache_key_value_strlen_callback(void *cnt, const char *key, const char *value) -{ -#endif - *((int*)cnt) += strlen(key) + 2 + ((value && *value) ? strlen(value) : 0); - return 1; +typedef struct header_cb_struct{ + apr_pool_t *pool; + char *str; +} header_cb_struct; + +/* Converts an integer value to its hex character*/ +char to_hex(char code) { + static char hex[] = "0123456789abcdef"; + return hex[code & 15]; +} + +/* Returns a url-encoded version of str */ +char *url_encode(apr_pool_t *p, const char *str) { + char *buf = apr_pcalloc(p, strlen(str) * 3 + 1), *pbuf = buf; + while (*str) { + if (isalnum(*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~') + *pbuf++ = *str; + else if (*str == ' ') + *pbuf++ = '+'; + else + *pbuf++ = '%', *pbuf++ = to_hex(*str >> 4), *pbuf++ = to_hex(*str & 15); + str++; + } + *pbuf = '\0'; + return buf; } #ifdef _WIN32 @@ -163,13 +212,15 @@ static APR_DECLARE_NONSTD(int) _mapcache_key_value_append_callback(void *cnt, const char *key, const char *value) { #endif -#define _mystr *((char**)cnt) - _mystr = apr_cpystrn(_mystr,key,MAX_STRING_LEN); - *((_mystr)++) = '='; +#define _mystr (((header_cb_struct*)cnt)->str) + header_cb_struct *hcs = (header_cb_struct*)cnt; + hcs->str = apr_pstrcat(hcs->pool, hcs->str, key, "=", NULL); if(value && *value) { - _mystr = apr_cpystrn(_mystr,value,MAX_STRING_LEN); + hcs->str = apr_pstrcat(hcs->pool, hcs->str, url_encode(hcs->pool, value), "&", NULL); + } + else { + hcs->str = apr_pstrcat(hcs->pool, hcs->str, "&", NULL); } - *((_mystr)++) = '&'; return 1; #undef _mystr } @@ -230,38 +281,29 @@ -char* mapcache_http_build_url(mapcache_context *r, char *base, apr_table_t *params) +char* mapcache_http_build_url(mapcache_context *ctx, char *base, apr_table_t *params) { if(!apr_is_empty_table(params)) { - int stringLength = 0, baseLength; - char *builtUrl,*builtUrlPtr; - char charToAppend=0; + int baseLength; + header_cb_struct hcs; baseLength = strlen(base); - - /*calculate the length of the param string we are going to build */ - apr_table_do(_mapcache_key_value_strlen_callback, (void*)&stringLength, params, NULL); + hcs.pool = ctx->pool; + hcs.str = base; if(strchr(base,'?')) { /* base already contains a '?' , shall we be adding a '&' to the end */ if(base[baseLength-1] != '?' && base[baseLength-1] != '&') { - charToAppend = '&'; + hcs.str = apr_pstrcat(ctx->pool, hcs.str, "&", NULL); } } else { /* base does not contain a '?', we will be adding it */ - charToAppend='?'; + hcs.str = apr_pstrcat(ctx->pool, hcs.str, "?", NULL); } - /* add final \0 and eventual separator to add ('?' or '&') */ - stringLength += baseLength + ((charToAppend)?2:1); - - builtUrl = builtUrlPtr = apr_palloc(r->pool, stringLength); - - builtUrlPtr = apr_cpystrn(builtUrlPtr,base,MAX_STRING_LEN); - if(charToAppend) - *(builtUrlPtr++)=charToAppend; - apr_table_do(_mapcache_key_value_append_callback, (void*)&builtUrlPtr, params, NULL); - *(builtUrlPtr-1) = '\0'; /*replace final '&' by a \0 */ - return builtUrl; + apr_table_do(_mapcache_key_value_append_callback, (void*)&hcs, params, NULL); + baseLength = strlen(hcs.str); + hcs.str[baseLength-1] = '\0'; + return hcs.str; } else { return base; } diff -Nru mapcache-1.2.1/lib/image.c mapcache-1.4.1/lib/image.c --- mapcache-1.2.1/lib/image.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/image.c 2016-02-25 15:47:16.000000000 +0000 @@ -85,6 +85,9 @@ void mapcache_image_merge(mapcache_context *ctx, mapcache_image *base, mapcache_image *overlay) { int starti,startj; + pixman_image_t *si; + pixman_image_t *bi; + pixman_transform_t transform; #ifndef USE_PIXMAN int i,j; unsigned char *browptr, *orowptr, *bptr, *optr; @@ -97,11 +100,10 @@ starti = (base->h - overlay->h)/2; startj = (base->w - overlay->w)/2; #ifdef USE_PIXMAN - pixman_image_t *si = pixman_image_create_bits(PIXMAN_a8r8g8b8,overlay->w,overlay->h, + si = pixman_image_create_bits(PIXMAN_a8r8g8b8,overlay->w,overlay->h, (uint32_t*)overlay->data,overlay->stride); - pixman_image_t *bi = pixman_image_create_bits(PIXMAN_a8r8g8b8,base->w,base->h, + bi = pixman_image_create_bits(PIXMAN_a8r8g8b8,base->w,base->h, (uint32_t*)base->data,base->stride); - pixman_transform_t transform; pixman_transform_init_translate(&transform, pixman_int_to_fixed(-startj), pixman_int_to_fixed(-starti)); @@ -359,5 +361,24 @@ return MAPCACHE_FALSE; } + +void mapcache_image_fill(mapcache_context *ctx, mapcache_image *image, const unsigned char *fill_color) { +#if 0 && defined(USE_PIXMAN) + pixman_fill((uint32_t*)image->data, image->stride, 32, 0, 0, image->w, image->h, *((int*)fill_color) ); +#else + int r,c; + unsigned char *pixptr; + for(r=0;rh;r++) { + pixptr = image->data + image->stride * r; + for(c=0;cw;c++) { + pixptr[0]=fill_color[0]; + pixptr[1]=fill_color[1]; + pixptr[2]=fill_color[2]; + pixptr[3]=fill_color[3]; + pixptr+=4; + } + } +#endif +} /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/lib/imageio_png.c mapcache-1.4.1/lib/imageio_png.c --- mapcache-1.2.1/lib/imageio_png.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/imageio_png.c 2016-02-25 15:47:16.000000000 +0000 @@ -45,6 +45,9 @@ #ifndef Z_BEST_COMPRESSION #define Z_BEST_COMPRESSION 9 #endif +#ifndef Z_NO_COMPRESSION +#define Z_NO_COMPRESSION 0 +#endif /* Table of CRCs of all 8-bit messages. */ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ @@ -146,48 +149,152 @@ return update_crc(0xffffffffL, buf, len) ^ 0xffffffffL; } -static unsigned char empty_png[] = { +static unsigned char empty_png_256[] = { 0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a,0x00,0x00,0x00,0x0d,0x49,0x48,0x44,0x52 ,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x03,0x00,0x00,0x00,0x66,0xbc,0x3a - ,0x25,0x00,0x00,0x00,0x03,0x50,0x4c,0x54,0x45,0x73,0x91,0xad,0x31,0xf0,0x8f,0xdd - ,0x00,0x00,0x00,0x01,0x74,0x52,0x4e,0x53,0xff,0x6d,0xe4,0x37,0xeb,0x00,0x00,0x00 + ,0x25,0x00,0x00,0x00,0x03, + /*PLTE*/0x50,0x4c,0x54,0x45, + /*RGB*/0x73,0x91,0xad, + /*PLTE-CRC*/0x31,0xf0,0x8f,0xdd + ,0x00,0x00,0x00,0x01, + /*tRNS*/0x74,0x52,0x4e,0x53, + /*A (fully opaque by default)*/0xff, + /*tRNS-CRC*/0x6d,0xe4,0x37,0xeb, + 0x00,0x00,0x00 ,0x1f,0x49,0x44,0x41,0x54,0x68,0xde,0xed,0xc1,0x01,0x0d,0x00,0x00,0x00,0xc2,0xa0 ,0xf7,0x4f,0x6d,0x0e,0x37,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xbe,0x0d ,0x21,0x00,0x00,0x01,0x7f,0x19,0x9c,0xa7,0x00,0x00,0x00,0x00,0x49,0x45,0x4e,0x44 ,0xae,0x42,0x60,0x82 }; +static size_t plte_offset_256= 0x25; +static size_t trns_offset_256 = 0x34; + + +unsigned char empty_png_384[] = { + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, + 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x01, 0x80, + 0x01, 0x03, 0x00, 0x00, 0x00, 0x9e, 0x09, 0x27, 0xfc, 0x00, 0x00, 0x00, + 0x03, + /*PLTE*/0x50,0x4c,0x54,0x45, + /*RGB*/0xcd,0x44,0x44, + /*PLTE-CRC*/0xdb,0xa9,0x37,0x41 + ,0x00,0x00,0x00,0x01, + /*tRNS*/0x74,0x52,0x4e,0x53, + /*A (fully opaque by default)*/0xff, + /*tRNS-CRC*/0x6d,0xe4,0x37,0xeb, + 0x00,0x00,0x00, + 0x28, 0x49, 0x44, 0x41, 0x54, 0x78, 0x5e, 0xed, + 0xc1, 0x01, 0x01, 0x00, 0x00, 0x00, 0x82, 0x20, 0xff, 0xaf, 0x6e, 0x48, + 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x19, 0x49, 0x80, 0x00, + 0x01, 0x28, 0x77, 0xfb, 0x61, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, + 0x44, 0xae, 0x42, 0x60, 0x82 +}; +static size_t plte_offset_384= 0x25; +static size_t trns_offset_384 = 0x34; + + +static unsigned char empty_png_512[] = { + 0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a,0x00,0x00,0x00,0x0d,0x49,0x48,0x44,0x52 + ,0x00,0x00,0x02,0x00,0x00,0x00,0x02,0x00,0x01,0x03,0x00,0x00,0x00,0xce,0xb6,0x46 + ,0xb9,0x00,0x00,0x00,0x03, + /*PLTE*/0x50,0x4c,0x54,0x45, + /*RGB*/0xcd,0x44,0x44, + /*PLTE-CRC*/0xdb,0xa9,0x37,0x41 + ,0x00,0x00,0x00,0x01, + /*tRNS*/0x74,0x52,0x4e,0x53, + /*A (fully opaque by default)*/0xff, + /*tRNS-CRC*/0x6d,0xe4,0x37,0xeb, + 0x00,0x00,0x00 + ,0x36,0x49,0x44,0x41,0x54,0x78,0x5e,0xed,0xc1,0x01,0x01,0x00,0x00,0x00,0x82,0x20 + ,0xff,0xaf,0x6e,0x48,0x40,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 + ,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 + ,0x00,0x00,0x00,0x00,0x00,0x7c,0x1b,0x82,0x00,0x00,0x01,0x9a,0x38,0x6a,0xc7,0x00 + ,0x00,0x00,0x00,0x49,0x45,0x4e,0x44,0xae,0x42,0x60,0x82 +}; +static size_t plte_offset_512= 0x25; +static size_t trns_offset_512 = 0x34; -static size_t plte_offset = 0x25; -static size_t trns_offset = 0x34; -mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, const unsigned char *hex_color, int *is_empty) { +mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, int width, int height, const unsigned char *hex_color, int *is_empty) { int chunkcrc; unsigned char *dd; - mapcache_buffer *encoded_data = mapcache_buffer_create(sizeof(empty_png)+4,ctx->pool); - dd = encoded_data->buf; - memcpy(dd,empty_png,sizeof(empty_png)); - dd[plte_offset+4] = hex_color[3]; //r - dd[plte_offset+5] = hex_color[2]; //g - dd[plte_offset+6] = hex_color[1]; //b - chunkcrc = crc(dd+plte_offset,7); - dd[plte_offset+7] = (unsigned char)((chunkcrc >> 24) & 0xff); - dd[plte_offset+8] = (unsigned char)((chunkcrc >> 16) & 0xff); - dd[plte_offset+9] = (unsigned char)((chunkcrc >> 8) & 0xff); - dd[plte_offset+10] = (unsigned char)(chunkcrc & 0xff); - if(hex_color[4] != 255) { - dd[trns_offset+4] = hex_color[4]; - chunkcrc = crc(dd+trns_offset,5); - dd[trns_offset+5] = (unsigned char)((chunkcrc >> 24) & 0xff); - dd[trns_offset+6] = (unsigned char)((chunkcrc >> 16) & 0xff); - dd[trns_offset+7] = (unsigned char)((chunkcrc >> 8) & 0xff); - dd[trns_offset+8] = (unsigned char)(chunkcrc & 0xff); + mapcache_buffer *encoded_data; + if(width == 256 && height == 256) { + encoded_data = mapcache_buffer_create(sizeof(empty_png_256)+4,ctx->pool); + dd = encoded_data->buf; + memcpy(dd,empty_png_256,sizeof(empty_png_256)); + dd[plte_offset_256+4] = hex_color[3]; //r + dd[plte_offset_256+5] = hex_color[2]; //g + dd[plte_offset_256+6] = hex_color[1]; //b + chunkcrc = crc(dd+plte_offset_256,7); + dd[plte_offset_256+7] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[plte_offset_256+8] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[plte_offset_256+9] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[plte_offset_256+10] = (unsigned char)(chunkcrc & 0xff); + if(hex_color[4] != 255) { + dd[trns_offset_256+4] = hex_color[4]; + chunkcrc = crc(dd+trns_offset_256,5); + dd[trns_offset_256+5] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[trns_offset_256+6] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[trns_offset_256+7] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[trns_offset_256+8] = (unsigned char)(chunkcrc & 0xff); + } + encoded_data->size = sizeof(empty_png_256); + } else if(width == 384 && height == 384) { + encoded_data = mapcache_buffer_create(sizeof(empty_png_384)+4,ctx->pool); + dd = encoded_data->buf; + memcpy(dd,empty_png_384,sizeof(empty_png_384)); + dd[plte_offset_384+4] = hex_color[3]; //r + dd[plte_offset_384+5] = hex_color[2]; //g + dd[plte_offset_384+6] = hex_color[1]; //b + chunkcrc = crc(dd+plte_offset_384,7); + dd[plte_offset_384+7] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[plte_offset_384+8] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[plte_offset_384+9] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[plte_offset_384+10] = (unsigned char)(chunkcrc & 0xff); + if(hex_color[4] != 255) { + dd[trns_offset_384+4] = hex_color[4]; + chunkcrc = crc(dd+trns_offset_384,5); + dd[trns_offset_384+5] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[trns_offset_384+6] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[trns_offset_384+7] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[trns_offset_384+8] = (unsigned char)(chunkcrc & 0xff); + } + encoded_data->size = sizeof(empty_png_384); + } else if(width == 512 && height == 512) { + encoded_data = mapcache_buffer_create(sizeof(empty_png_512)+4,ctx->pool); + dd = encoded_data->buf; + memcpy(dd,empty_png_512,sizeof(empty_png_512)); + dd[plte_offset_512+4] = hex_color[3]; //r + dd[plte_offset_512+5] = hex_color[2]; //g + dd[plte_offset_512+6] = hex_color[1]; //b + chunkcrc = crc(dd+plte_offset_512,7); + dd[plte_offset_512+7] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[plte_offset_512+8] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[plte_offset_512+9] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[plte_offset_512+10] = (unsigned char)(chunkcrc & 0xff); + if(hex_color[4] != 255) { + dd[trns_offset_512+4] = hex_color[4]; + chunkcrc = crc(dd+trns_offset_512,5); + dd[trns_offset_512+5] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[trns_offset_512+6] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[trns_offset_512+7] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[trns_offset_512+8] = (unsigned char)(chunkcrc & 0xff); + } + encoded_data->size = sizeof(empty_png_512); + } else { + /* here for compatibility reasons, although this should not be used in production as it is cpu-heavy */ + mapcache_image *rgba = mapcache_image_create_with_data(ctx,width,height); + mapcache_image_format *format = mapcache_configuration_get_image_format(ctx->config,"PNG8"); + mapcache_image_fill(ctx,rgba,hex_color+1); + encoded_data = format->write(ctx,rgba,format); } if(hex_color[4] == 0) { *is_empty = 1; } else { *is_empty = 0; } - encoded_data->size = sizeof(empty_png); return encoded_data; } @@ -217,14 +324,10 @@ // do nothing } -#ifndef _WIN32 -static inline int premultiply (int color,int alpha) -#else -static __inline int premultiply (int color,int alpha) -#endif -{ - int temp = (alpha * color) + 0x80; - return ((temp + (temp >> 8)) >> 8); +#define PREMULTIPLY(out,color,alpha)\ +{\ + int temp = ((alpha) * (color)) + 0x80;\ + out =((temp + (temp >> 8)) >> 8);\ } @@ -318,9 +421,9 @@ pixptr[1] = 0; pixptr[2] = 0; } else { - pixptr[0] = premultiply(pixel[2],alpha); - pixptr[1] = premultiply(pixel[1],alpha); - pixptr[2] = premultiply(pixel[0],alpha); + PREMULTIPLY(pixptr[0],pixel[2],alpha); + PREMULTIPLY(pixptr[1],pixel[1],alpha); + PREMULTIPLY(pixptr[2],pixel[0],alpha); } pixptr += 4; } @@ -410,6 +513,9 @@ png_set_compression_level (png_ptr, Z_BEST_COMPRESSION); else if(compression == MAPCACHE_COMPRESSION_FAST) png_set_compression_level (png_ptr, Z_BEST_SPEED); + else if(compression == MAPCACHE_COMPRESSION_DISABLE) + png_set_compression_level (png_ptr, Z_NO_COMPRESSION); + png_set_filter(png_ptr,0,PNG_FILTER_NONE); info_ptr = png_create_info_struct(png_ptr); @@ -1314,6 +1420,8 @@ png_set_compression_level (png_ptr, Z_BEST_COMPRESSION); else if(compression == MAPCACHE_COMPRESSION_FAST) png_set_compression_level (png_ptr, Z_BEST_SPEED); + else if(compression == MAPCACHE_COMPRESSION_DISABLE) + png_set_compression_level (png_ptr, Z_NO_COMPRESSION); png_set_filter(png_ptr,0,PNG_FILTER_NONE); info_ptr = png_create_info_struct(png_ptr); if (!info_ptr) { @@ -1413,7 +1521,7 @@ mapcache_image_format_png_q *format = apr_pcalloc(pool, sizeof(mapcache_image_format_png_q)); format->format.format.name = name; format->format.format.extension = apr_pstrdup(pool,"png"); - format->format.format.mime_type = apr_pstrdup(pool,"image/png"); + format->format.format.mime_type = apr_pstrdup(pool,"image/png; mode=8bit"); format->format.compression_level = compression; format->format.format.write = _mapcache_imageio_png_q_encode; format->format.format.create_empty_image = _mapcache_imageio_png_create_empty; diff -Nru mapcache-1.2.1/lib/lock.c mapcache-1.4.1/lib/lock.c --- mapcache-1.2.1/lib/lock.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/lock.c 2016-02-25 15:47:16.000000000 +0000 @@ -31,8 +31,13 @@ #include #include #include +#ifndef _WIN32 +#include +#endif + +#define MAPCACHE_LOCKFILE_PREFIX "_gc_lock" -char* lock_filename_for_resource(mapcache_context *ctx, const char *resource) +char* lock_filename_for_resource(mapcache_context *ctx, mapcache_locker_disk *ldisk, const char *resource) { char *saferes = apr_pstrdup(ctx->pool,resource); char *safeptr = saferes; @@ -43,45 +48,373 @@ safeptr++; } return apr_psprintf(ctx->pool,"%s/"MAPCACHE_LOCKFILE_PREFIX"%s.lck", - ctx->config->lockdir,saferes); + ldisk->dir,saferes); } -int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, char *resource) +int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void **lock) { - char *lockname = lock_filename_for_resource(ctx,resource); + mapcache_lock_result rv = locker->aquire_lock(ctx, locker, resource, lock); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + if(rv == MAPCACHE_LOCK_AQUIRED) + return MAPCACHE_TRUE; + else { + apr_time_t start_wait = apr_time_now(); + rv = MAPCACHE_LOCK_LOCKED; + + while(rv != MAPCACHE_LOCK_NOENT) { + unsigned int waited = apr_time_as_msec(apr_time_now()-start_wait); + if(waited > locker->timeout*1000) { + mapcache_unlock_resource(ctx,locker,resource, *lock); + ctx->log(ctx,MAPCACHE_ERROR,"deleting a possibly stale lock after waiting on it for %g seconds",waited/1000.0); + return MAPCACHE_FALSE; + } + apr_sleep(locker->retry_interval * 1000000); + rv = locker->ping_lock(ctx,locker,resource, *lock); + } + return MAPCACHE_FALSE; + } +} + + +mapcache_lock_result mapcache_locker_disk_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { + char *lockname, errmsg[120]; + mapcache_locker_disk *ldisk; apr_file_t *lockfile; apr_status_t rv; + + assert(self->type == MAPCACHE_LOCKER_DISK); + ldisk = (mapcache_locker_disk*)self; + *lock = NULL; /*unused*/ + + lockname = lock_filename_for_resource(ctx,ldisk,resource); /* create the lockfile */ rv = apr_file_open(&lockfile,lockname,APR_WRITE|APR_CREATE|APR_EXCL|APR_XTHREAD,APR_OS_DEFAULT,ctx->pool); /* if the file already exists, wait for it to disappear */ /* TODO: check the lock isn't stale (i.e. too old) */ if( rv != APR_SUCCESS ) { - apr_finfo_t info; - rv = apr_stat(&info,lockname,0,ctx->pool); -#ifdef DEBUG - if(!APR_STATUS_IS_ENOENT(rv)) { - ctx->log(ctx, MAPCACHE_DEBUG, "waiting on resource lock %s", resource); + if( !APR_STATUS_IS_EEXIST(rv) ) { + ctx->set_error(ctx, 500, "failed to create lockfile %s: %s", lockname, apr_strerror(rv,errmsg,120)); + return MAPCACHE_LOCK_NOENT; } -#endif - while(!APR_STATUS_IS_ENOENT(rv)) { - /* sleep for the configured number of micro-seconds (default is 1/100th of a second) */ - apr_sleep(ctx->config->lock_retry_interval); - rv = apr_stat(&info,lockname,0,ctx->pool); - } - return MAPCACHE_FALSE; + return MAPCACHE_LOCK_LOCKED; } else { /* we acquired the lock */ + char *pid_s; + pid_t pid; + apr_size_t pid_s_len; + pid = getpid(); + pid_s = apr_psprintf(ctx->pool,"%"APR_PID_T_FMT,pid); + pid_s_len = strlen(pid_s); + apr_file_write(lockfile,pid_s,&pid_s_len); apr_file_close(lockfile); - return MAPCACHE_TRUE; + return MAPCACHE_LOCK_AQUIRED; } } -void mapcache_unlock_resource(mapcache_context *ctx, char *resource) +mapcache_lock_result mapcache_locker_disk_ping_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { + apr_finfo_t info; + apr_status_t rv; + char *lockname; + mapcache_locker_disk *ldisk = (mapcache_locker_disk*)self; + lockname = lock_filename_for_resource(ctx,ldisk,resource); + rv = apr_stat(&info,lockname,0,ctx->pool); + if(APR_STATUS_IS_ENOENT(rv)) { + return MAPCACHE_LOCK_NOENT; + } else { + return MAPCACHE_LOCK_LOCKED; + } +} + +void mapcache_locker_disk_release_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { - char *lockname = lock_filename_for_resource(ctx,resource); + mapcache_locker_disk *ld = (mapcache_locker_disk*)self; + char *lockname = lock_filename_for_resource(ctx,ld,resource); apr_file_remove(lockname,ctx->pool); } +void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void *lock) { + locker->release_lock(ctx, locker, resource, lock); +} + +void mapcache_locker_disk_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { + mapcache_locker_disk *ldisk = (mapcache_locker_disk*)self; + ezxml_t node; + if((node = ezxml_child(doc,"directory")) != NULL) { + ldisk->dir = apr_pstrdup(ctx->pool, node->txt); + } else { + ldisk->dir = apr_pstrdup(ctx->pool,"/tmp"); + } +} + +mapcache_locker* mapcache_locker_disk_create(mapcache_context *ctx) { + mapcache_locker_disk *ld = (mapcache_locker_disk*)apr_pcalloc(ctx->pool, sizeof(mapcache_locker_disk)); + mapcache_locker *l = (mapcache_locker*)ld; + l->type = MAPCACHE_LOCKER_DISK; + l->aquire_lock = mapcache_locker_disk_aquire_lock; + l->parse_xml = mapcache_locker_disk_parse_xml; + l->release_lock = mapcache_locker_disk_release_lock; + l->ping_lock = mapcache_locker_disk_ping_lock; + return l; +} + +struct mapcache_locker_fallback_lock { + mapcache_locker *locker; /*the locker that actually acquired the lock*/ + void *lock; /*the opaque lock returned by the locker*/ +}; + +void mapcache_locker_fallback_release_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { + struct mapcache_locker_fallback_lock *flock = lock; + flock->locker->release_lock(ctx,flock->locker,resource,flock->lock); +} + +mapcache_lock_result mapcache_locker_fallback_ping_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { + struct mapcache_locker_fallback_lock *flock = lock; + return flock->locker->ping_lock(ctx,flock->locker,resource,flock->lock); +} + +mapcache_lock_result mapcache_locker_fallback_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { + int i; + mapcache_locker_fallback *locker = (mapcache_locker_fallback*)self; + struct mapcache_locker_fallback_lock *fallback_lock = apr_pcalloc(ctx->pool, sizeof(struct mapcache_locker_fallback_lock)); + *lock = fallback_lock; + for(i=0;ilockers->nelts;i++) { + mapcache_lock_result lock_result; + mapcache_locker *child_locker = APR_ARRAY_IDX(locker->lockers, i, mapcache_locker*); + void *error; + ctx->pop_errors(ctx,&error); + lock_result = child_locker->aquire_lock(ctx, child_locker, resource, &(fallback_lock->lock)); + if(!GC_HAS_ERROR(ctx)) { + fallback_lock->locker = child_locker; + ctx->push_errors(ctx,error); + return lock_result; + } else { + /*clear the current error if we still have a fallback lock to try */ + if(ilockers->nelts-1) { + ctx->clear_errors(ctx); + } + } + ctx->push_errors(ctx,error); + } + return MAPCACHE_LOCK_NOENT; +} + + +void mapcache_locker_fallback_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { + mapcache_locker_fallback *lm = (mapcache_locker_fallback*)self; + ezxml_t node; + lm->lockers = apr_array_make(ctx->pool,2,sizeof(mapcache_locker*)); + for(node = ezxml_child(doc,"locker"); node; node = node->next) { + mapcache_locker *child_locker; + mapcache_config_parse_locker(ctx,node,&child_locker); + GC_CHECK_ERROR(ctx); + APR_ARRAY_PUSH(lm->lockers,mapcache_locker*) = child_locker; + } +} + +#ifdef USE_MEMCACHE +void mapcache_locker_memcache_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + ezxml_t node,server_node; + char *endptr; + for(server_node = ezxml_child(doc,"server"); server_node; server_node = server_node->next) { + lm->nservers++; + } + lm->servers = apr_pcalloc(ctx->pool, lm->nservers * sizeof(mapcache_locker_memcache_server)); + lm->nservers = 0; + for(server_node = ezxml_child(doc,"server"); server_node; server_node = server_node->next) { + if((node = ezxml_child(server_node,"host")) != NULL) { + lm->servers[lm->nservers].host = apr_pstrdup(ctx->pool, node->txt); + } else { + ctx->set_error(ctx, 400, "memcache locker: no provided"); + return; + } + + if((node = ezxml_child(server_node,"port")) != NULL) { + lm->servers[lm->nservers].port = (unsigned int)strtol(node->txt,&endptr,10); + if(*endptr != 0 || lm->servers[lm->nservers].port <= 0) { + ctx->set_error(ctx, 400, "failed to parse memcache locker port \"%s\". Expecting a positive integer", + node->txt); + return; + } + } else { + /* default memcached port */ + lm->servers[lm->nservers].port = 11211; + } + lm->nservers++; + } +} + +static char* memcache_key_for_resource(mapcache_context *ctx, mapcache_locker_memcache *lm, const char *resource) +{ + char *saferes = apr_pstrdup(ctx->pool,resource); + char *safeptr = saferes; + while(*safeptr) { + if(*safeptr==' ' || *safeptr == '/' || *safeptr == '~' || *safeptr == '.' || + *safeptr == '\r' || *safeptr == '\n' || *safeptr == '\t' || *safeptr == '\f' || *safeptr == '\e' || *safeptr == '\a' || *safeptr == '\b') { + *safeptr = '#'; + } + safeptr++; + } + return apr_psprintf(ctx->pool,MAPCACHE_LOCKFILE_PREFIX"%s.lck",saferes); +} + +apr_memcache_t* create_memcache(mapcache_context *ctx, mapcache_locker_memcache *lm) { + apr_status_t rv; + apr_memcache_t *memcache; + char errmsg[120]; + int i; + if(APR_SUCCESS != apr_memcache_create(ctx->pool, lm->nservers, 0, &memcache)) { + ctx->set_error(ctx,500,"memcache locker: failed to create memcache backend"); + return NULL; + } + + for(i=0;inservers;i++) { + apr_memcache_server_t *server; + rv = apr_memcache_server_create(ctx->pool,lm->servers[i].host,lm->servers[i].port,1,1,1,10000,&server); + if(APR_SUCCESS != rv) { + ctx->set_error(ctx,500,"memcache locker: failed to create server %s:%d: %s",lm->servers[i].host,lm->servers[i].port, apr_strerror(rv,errmsg,120)); + return NULL; + } + + rv = apr_memcache_add_server(memcache,server); + if(APR_SUCCESS != rv) { + ctx->set_error(ctx,500,"memcache locker: failed to add server %s:%d: %s",lm->servers[i].host,lm->servers[i].port, apr_strerror(rv,errmsg,120)); + return NULL; + } + } + return memcache; +} + +mapcache_lock_result mapcache_locker_memcache_ping_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { + apr_status_t rv; + char *one; + size_t ione; + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + char *key = memcache_key_for_resource(ctx, lm, resource); + apr_memcache_t *memcache = (apr_memcache_t*)lock; + if(!memcache) + return MAPCACHE_LOCK_NOENT; + rv = apr_memcache_getp(memcache,ctx->pool,key,&one,&ione,NULL); + if(rv == APR_SUCCESS) + return MAPCACHE_LOCK_LOCKED; + else + return MAPCACHE_LOCK_NOENT; +} + + +mapcache_lock_result mapcache_locker_memcache_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { + apr_status_t rv; + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + char errmsg[120]; + char *key = memcache_key_for_resource(ctx, lm, resource); + apr_memcache_t *memcache = create_memcache(ctx,lm); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_LOCK_NOENT; + } + *lock = memcache; + rv = apr_memcache_add(memcache,key,"1",1,self->timeout,0); + if( rv == APR_SUCCESS) { + return MAPCACHE_LOCK_AQUIRED; + } else if ( rv == APR_EEXIST ) { + return MAPCACHE_LOCK_LOCKED; + } else { + ctx->set_error(ctx,500,"failed to lock resource %s to memcache locker: %s",resource, apr_strerror(rv,errmsg,120)); + return MAPCACHE_LOCK_NOENT; + } +} + +void mapcache_locker_memcache_release_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { + apr_status_t rv; + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + char errmsg[120]; + char *key = memcache_key_for_resource(ctx, lm, resource); + apr_memcache_t *memcache = (apr_memcache_t*)lock; + if(!memcache) { + /*error*/ + return; + } + + rv = apr_memcache_delete(memcache,key,0); + if(rv != APR_SUCCESS && rv!= APR_NOTFOUND) { + ctx->set_error(ctx,500,"memcache: failed to delete key %s: %s", key, apr_strerror(rv,errmsg,120)); + } + +} + +mapcache_locker* mapcache_locker_memcache_create(mapcache_context *ctx) { + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)apr_pcalloc(ctx->pool, sizeof(mapcache_locker_memcache)); + mapcache_locker *l = (mapcache_locker*)lm; + l->type = MAPCACHE_LOCKER_MEMCACHE; + l->aquire_lock = mapcache_locker_memcache_aquire_lock; + l->ping_lock = mapcache_locker_memcache_ping_lock; + l->parse_xml = mapcache_locker_memcache_parse_xml; + l->release_lock = mapcache_locker_memcache_release_lock; + lm->nservers = 0; + lm->servers = NULL; + return l; +} + +#endif + +mapcache_locker* mapcache_locker_fallback_create(mapcache_context *ctx) { + mapcache_locker_fallback *lm = (mapcache_locker_fallback*)apr_pcalloc(ctx->pool, sizeof(mapcache_locker_fallback)); + mapcache_locker *l = (mapcache_locker*)lm; + l->type = MAPCACHE_LOCKER_FALLBACK; + l->aquire_lock = mapcache_locker_fallback_aquire_lock; + l->ping_lock = mapcache_locker_fallback_ping_lock; + l->parse_xml = mapcache_locker_fallback_parse_xml; + l->release_lock = mapcache_locker_fallback_release_lock; + return l; +} + +void mapcache_config_parse_locker(mapcache_context *ctx, ezxml_t node, mapcache_locker **locker) { + ezxml_t cur_node; + const char *ltype = ezxml_attr(node, "type"); + if(!ltype) ltype = "disk"; + if(!strcmp(ltype,"disk")) { + *locker = mapcache_locker_disk_create(ctx); + } else if(!strcmp(ltype,"fallback")) { + *locker = mapcache_locker_fallback_create(ctx); + } else if(!strcmp(ltype,"memcache")) { +#ifdef USE_MEMCACHE + *locker = mapcache_locker_memcache_create(ctx); +#else + ctx->set_error(ctx,400,": type \"memcache\" cannot be used as memcache support is not compiled in"); + return; +#endif + } else { + ctx->set_error(ctx,400,": unknown type \"%s\" (allowed are disk and memcache)",ltype); + return; + } + (*locker)->parse_xml(ctx, *locker, node); + + if((cur_node = ezxml_child(node,"retry")) != NULL) { + char *endptr; + (*locker)->retry_interval = strtod(cur_node->txt,&endptr); + if(*endptr != 0 || (*locker)->retry_interval <= 0) { + ctx->set_error(ctx, 400, "failed to locker parse retry seconds \"%s\". Expecting a positive floating point number", + cur_node->txt); + return; + } + } else { + /* default retry interval is 1/10th of a second */ + (*locker)->retry_interval = 0.1; + } + if((cur_node = ezxml_child(node,"timeout")) != NULL) { + char *endptr; + (*locker)->timeout = strtod(cur_node->txt,&endptr); + if(*endptr != 0 || (*locker)->timeout <= 0) { + ctx->set_error(ctx, 400, "failed to parse locker timeout seconds \"%s\". Expecting a positive floating point number", + cur_node->txt); + return; + } + } else { + /* default timeout is 2 minutes */ + (*locker)->timeout = 120; + } +} /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/lib/service_demo.c mapcache-1.4.1/lib/service_demo.c --- mapcache-1.2.1/lib/service_demo.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/service_demo.c 2016-02-25 15:47:16.000000000 +0000 @@ -553,10 +553,13 @@ apr_array_header_t *timedimvals = tileset->timedimension->get_all_entries( ctx,tileset->timedimension,tileset); for(id=0;idnelts;id++) { + char *idval; + char *dimparam_wms; + char *dimparam_singletile; if(id>1) break; - char *idval = APR_ARRAY_IDX(timedimvals,id,char*); - char *dimparam_wms = " %s_wms_layer.mergeNewParams({%s:\"%s\"});\n"; - char *dimparam_singletile = " %s_slayer.mergeNewParams({%s:\"%s\"});\n"; + idval = APR_ARRAY_IDX(timedimvals,id,char*); + dimparam_wms = " %s_wms_layer.mergeNewParams({%s:\"%s\"});\n"; + dimparam_singletile = " %s_slayer.mergeNewParams({%s:\"%s\"});\n"; ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s", tileset->name, grid->name, idval); /* normalize name to something that is a valid variable name */ for(i=0; ipool,"%s%s",caps,ol_layer); caps = apr_psprintf(ctx->pool,"%s%s",caps, apr_psprintf(ctx->pool,dimparam_wms,ol_layer_name,tileset->timedimension->key,idval)); - + if(service->getmap_strategy == MAPCACHE_GETMAP_ASSEMBLE) { ol_layer = apr_psprintf(ctx->pool, demo_layer_singletile, ol_layer_name, @@ -660,15 +663,11 @@ tileindex_index = apr_hash_first(ctx->pool,ctx->config->tilesets); while(tileindex_index) { int i,j; - char *extension; mapcache_tileset *tileset; const void *key; apr_ssize_t keylen; apr_hash_this(tileindex_index,&key,&keylen,(void**)&tileset); - extension = "png"; - if (tileset->format && tileset->format->extension) - extension = tileset->format->extension; for(j=0; jgrid_links->nelts; j++) { char *resolutions=""; char *unit="dd"; @@ -873,9 +872,11 @@ ctx,tileset->timedimension,tileset); GC_CHECK_ERROR(ctx); for(id=0;idnelts;id++) { + char *idval; + char *dimparam; if(id>1) break; /* we don't want all the entries in the demo interface */ - char *idval = APR_ARRAY_IDX(timedimvals,id,char*); - char *dimparam = "%s_wmts_layer.mergeNewParams({%s:\"%s\"});\n"; + idval = APR_ARRAY_IDX(timedimvals,id,char*); + dimparam = "%s_wmts_layer.mergeNewParams({%s:\"%s\"});\n"; ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s", tileset->name, grid->name, idval); /* normalize name to something that is a valid variable name */ for(i=0; ipool,"%s%s",caps,ol_layer); caps = apr_psprintf(ctx->pool,"%s%s",caps, apr_psprintf(ctx->pool,dimparam,ol_layer_name,tileset->timedimension->key,idval)); - + } } } diff -Nru mapcache-1.2.1/lib/service_mapguide.c mapcache-1.4.1/lib/service_mapguide.c --- mapcache-1.2.1/lib/service_mapguide.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/service_mapguide.c 2016-02-25 15:47:16.000000000 +0000 @@ -124,7 +124,7 @@ if(index == 5) { char *gridname; mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc(ctx->pool,sizeof(mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; gridname = sTileset; /*hijack the char* pointer while counting the number of commas */ while(*gridname) { if(*gridname == ';') req->ntiles++; diff -Nru mapcache-1.2.1/lib/service_tms.c mapcache-1.4.1/lib/service_tms.c --- mapcache-1.2.1/lib/service_tms.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/service_tms.c 2016-02-25 15:47:16.000000000 +0000 @@ -240,7 +240,7 @@ if(index == 5) { char *gridname; mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc(ctx->pool,sizeof(mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; gridname = sTileset; /*hijack the char* pointer while counting the number of commas */ while(*gridname) { if(*gridname == ';') req->ntiles++; diff -Nru mapcache-1.2.1/lib/service_ve.c mapcache-1.4.1/lib/service_ve.c --- mapcache-1.2.1/lib/service_ve.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/service_ve.c 2016-02-25 15:47:16.000000000 +0000 @@ -131,7 +131,7 @@ req = (mapcache_request_get_tile*) apr_pcalloc(ctx->pool, sizeof (mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; req->ntiles = 1; req->tiles = (mapcache_tile**) apr_pcalloc(ctx->pool, sizeof (mapcache_tile*)); req->tiles[0] = tile; diff -Nru mapcache-1.2.1/lib/service_wms.c mapcache-1.4.1/lib/service_wms.c --- mapcache-1.2.1/lib/service_wms.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/service_wms.c 2016-02-25 15:47:16.000000000 +0000 @@ -200,7 +200,7 @@ layerxml = ezxml_add_child(toplayer,"Layer",0); ezxml_set_attr(layerxml, "cascaded", "1"); ezxml_set_attr(layerxml, "queryable", (tileset->source && tileset->source->info_formats)?"1":"0"); - + ezxml_set_txt(ezxml_add_child(layerxml,"Name",0),tileset->name); tsxml = ezxml_add_child(vendorxml, "TileSet",0); @@ -228,8 +228,9 @@ if(tileset->dimensions) { for(i=0; idimensions->nelts; i++) { - const char **value; - char *dimval; + apr_array_header_t *values; + int value_idx; + char *dimval = NULL; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); ezxml_t dimxml = ezxml_add_child(layerxml,"Dimension",0); ezxml_set_attr(dimxml,"name",dimension->name); @@ -238,12 +239,14 @@ if(dimension->unit) { ezxml_set_attr(dimxml,"units",dimension->unit); } - value = dimension->print_ogc_formatted_values(ctx,dimension); - dimval = apr_pstrdup(ctx->pool,*value); - value++; - while(*value) { - dimval = apr_pstrcat(ctx->pool,dimval,",",*value,NULL); - value++; + values = dimension->print_ogc_formatted_values(ctx,dimension); + for(value_idx=0;value_idxnelts;value_idx++) { + char *idval = APR_ARRAY_IDX(values,value_idx,char*); + if(dimval) { + dimval = apr_pstrcat(ctx->pool,dimval,",",idval,NULL); + } else { + dimval = apr_pstrdup(ctx->pool,idval); + } } ezxml_set_txt(dimxml,dimval); } @@ -483,6 +486,7 @@ mapcache_grid_link *main_grid_link = NULL; mapcache_tileset *main_tileset = NULL; mapcache_request_type type; + mapcache_image_format *imf; /* count the number of layers that are requested. * if we are in combined-mirror mode, then there is @@ -545,24 +549,44 @@ type = MAPCACHE_REQUEST_GET_MAP; } + imf = wms_service->getmap_format; + if(wms_service->allow_format_override) { + str = apr_table_get(params,"FORMAT"); + if(strcmp(str,imf->name) && strcmp(str,imf->mime_type)) { + apr_hash_index_t *hi; + for (hi = apr_hash_first(ctx->pool, ctx->config->image_formats); hi; hi = apr_hash_next(hi)) { + apr_hash_this(hi, NULL, NULL, (void**)&imf); + if(!strcmp(imf->name, str) || (imf->mime_type && !strcmp(imf->mime_type, str))) { + break; + } + } + if(!hi) { /* did not find any matching format for given mimetype or name */ + errcode = 404; + errmsg = apr_psprintf(ctx->pool,"received wms request with invalid format %s", str); + goto proxies; + } + } + } + if(type == MAPCACHE_REQUEST_GET_TILE) { tile_req = apr_pcalloc(ctx->pool, sizeof(mapcache_request_get_tile)); - tile_req->request.type = MAPCACHE_REQUEST_GET_TILE; tile_req->tiles = apr_pcalloc(ctx->pool, count*sizeof(mapcache_tile*)); - tile_req->format = wms_service->getmap_format; + tile_req->image_request.format = imf; *request = (mapcache_request*)tile_req; + (*request)->type = MAPCACHE_REQUEST_GET_TILE; } else { map_req = apr_pcalloc(ctx->pool, sizeof(mapcache_request_get_map)); - map_req->request.type = MAPCACHE_REQUEST_GET_MAP; map_req->maps = apr_pcalloc(ctx->pool, count*sizeof(mapcache_map*)); map_req->getmap_strategy = wms_service->getmap_strategy; map_req->resample_mode = wms_service->resample_mode; - map_req->getmap_format = wms_service->getmap_format; + map_req->image_request.format = imf; *request = (mapcache_request*)map_req; + (*request)->type = MAPCACHE_REQUEST_GET_MAP; } nallocated = count; + /* * loop through all the layers to verify that they reference the requested grid, * and to extract any dimensions if configured @@ -636,17 +660,36 @@ const char *value; if(tileset->dimensions) { for(i=0; idimensions->nelts; i++) { + char *dim_name; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - if((value = (char*)apr_table_get(params,dimension->name)) != NULL) { + if(!strcasecmp(dimension->name,"TIME") || !strcasecmp(dimension->name,"ELEVATION")) { + dim_name = dimension->name; + } else { + dim_name = apr_pstrcat(ctx->pool, "dim_", dimension->name, NULL); + } + if((value = (char*)apr_table_get(params,dim_name)) == NULL) { + if(strcasecmp(dimension->name,"TIME") && strcasecmp(dimension->name,"ELEVATION")) { + /* also test for the dimension without the DIM_ prefix if the latter was not found in the KVP params */ + dim_name = dimension->name; + value = (char*)apr_table_get(params,dimension->name); + } + } + + if(value) { char *tmpval = apr_pstrdup(ctx->pool,value); - int ok = dimension->validate(ctx,dimension,&tmpval); - GC_CHECK_ERROR(ctx); + int ok; + if(dimension->skip_validation) { + ok = MAPCACHE_SUCCESS; + } else { + ok = dimension->validate(ctx,dimension,&tmpval); + GC_CHECK_ERROR(ctx); + } if(ok == MAPCACHE_SUCCESS) apr_table_setn(dimtable,dimension->name,tmpval); else { errcode = 400; errmsg = apr_psprintf(ctx->pool, "dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); + dim_name, value); goto proxies; } } @@ -672,9 +715,9 @@ /* we need to create more tile/map entries */ if(timedim_selected->nelts > 1) { /* apr pools have no realloc */ + mapcache_tile** tmptiles; nallocated = nallocated + timedim_selected->nelts - 1; - mapcache_tile** tmptiles = - apr_palloc(ctx->pool, nallocated * sizeof(mapcache_tile*)); + tmptiles = apr_palloc(ctx->pool, nallocated * sizeof(mapcache_tile*)); for(i=0;intiles;i++) { tmptiles[i] = tile_req->tiles[i]; } @@ -695,9 +738,9 @@ /* we need to create more tile/map entries */ if(timedim_selected->nelts > 1) { /* apr pools have no realloc */ + mapcache_map** tmpmaps; nallocated = nallocated + timedim_selected->nelts - 1; - mapcache_map** tmpmaps = - apr_palloc(ctx->pool, nallocated * sizeof(mapcache_map*)); + tmpmaps = apr_palloc(ctx->pool, nallocated * sizeof(mapcache_map*)); for(i=0;inmaps;i++) { tmpmaps[i] = map_req->maps[i]; } @@ -713,7 +756,7 @@ apr_table_set(map_req->maps[map_req->nmaps-1]->dimensions,tileset->timedimension->key, APR_ARRAY_IDX(timedim_selected,i,char*)); } - + } } } @@ -876,7 +919,7 @@ *request = (mapcache_request*)req_proxy; (*request)->service = this; (*request)->type = MAPCACHE_REQUEST_PROXY; - req_proxy->http = rule->http; + req_proxy->rule = rule; req_proxy->params = params; if(rule->append_pathinfo) { req_proxy->pathinfo = pathinfo; @@ -916,32 +959,42 @@ for( rule_node = ezxml_child(node,"forwarding_rule"); rule_node; rule_node = rule_node->next) { mapcache_forwarding_rule *rule; - ezxml_t http_node; - ezxml_t pathinfo_node; - ezxml_t param_node; + ezxml_t node; char *name = (char*)ezxml_attr(rule_node,"name"); if(!name) name = "(null)"; rule = apr_pcalloc(ctx->pool, sizeof(mapcache_forwarding_rule)); rule->name = apr_pstrdup(ctx->pool,name); rule->match_params = apr_array_make(ctx->pool,1,sizeof(mapcache_dimension*)); + rule->max_post_len = 10485760; /* 10 megabytes by default */ - pathinfo_node = ezxml_child(rule_node,"append_pathinfo"); - if(pathinfo_node && !strcasecmp(pathinfo_node->txt,"true")) { + node = ezxml_child(rule_node,"append_pathinfo"); + if(node && !strcasecmp(node->txt,"true")) { rule->append_pathinfo = 1; } else { rule->append_pathinfo = 0; } - http_node = ezxml_child(rule_node,"http"); - if(!http_node) { + + node = ezxml_child(rule_node,"max_post_length"); + if(node) { + char *endptr; + rule->max_post_len= (size_t)strtol(node->txt,&endptr,10); + if(*endptr != 0 || rule->max_post_len <= 0) { + ctx->set_error(ctx,500,"rule \"%s\" cannot have a negative or null ",name); + return; + } + } + + node = ezxml_child(rule_node,"http"); + if(!node) { ctx->set_error(ctx,500,"rule \"%s\" does not contain an block",name); return; } - rule->http = mapcache_http_configuration_parse_xml(ctx,http_node); + rule->http = mapcache_http_configuration_parse_xml(ctx,node); GC_CHECK_ERROR(ctx); - for(param_node = ezxml_child(rule_node,"param"); param_node; param_node = param_node->next) { - char *name = (char*)ezxml_attr(param_node,"name"); - char *type = (char*)ezxml_attr(param_node,"type"); + for(node = ezxml_child(rule_node,"param"); node; node = node->next) { + char *name = (char*)ezxml_attr(node,"name"); + char *type = (char*)ezxml_attr(node,"type"); mapcache_dimension *dimension = NULL; @@ -966,7 +1019,7 @@ dimension->name = apr_pstrdup(ctx->pool,name); - dimension->configuration_parse_xml(ctx,dimension,param_node); + dimension->configuration_parse_xml(ctx,dimension,node); GC_CHECK_ERROR(ctx); APR_ARRAY_PUSH(rule->match_params,mapcache_dimension*) = dimension; @@ -986,11 +1039,16 @@ wms->getmap_format = mapcache_configuration_get_image_format(cfg,"JPEG"); if ((rule_node = ezxml_child(node,"format")) != NULL) { + const char *attr; wms->getmap_format = mapcache_configuration_get_image_format(cfg,rule_node->txt); if(!wms->getmap_format) { ctx->set_error(ctx,400, "unknown %s for wms service", rule_node->txt); return; } + attr = ezxml_attr(rule_node,"allow_client_override"); + if(attr && !strcmp(attr,"true")) { + wms->allow_format_override = 1; + } } if ((rule_node = ezxml_child(node,"resample_mode")) != NULL) { @@ -1064,6 +1122,7 @@ service->getmap_strategy = MAPCACHE_GETMAP_ASSEMBLE; service->resample_mode = MAPCACHE_RESAMPLE_BILINEAR; service->getmap_format = NULL; + service->allow_format_override = 0; return (mapcache_service*)service; } diff -Nru mapcache-1.2.1/lib/service_wmts.c mapcache-1.4.1/lib/service_wmts.c --- mapcache-1.2.1/lib/service_wmts.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/service_wmts.c 2016-02-25 15:47:16.000000000 +0000 @@ -348,6 +348,15 @@ ezxml_set_txt(ezxml_add_child(layer,"ows:Abstract",0),abstract); } + if(tileset->wgs84bbox.minx != tileset->wgs84bbox.maxx) { + ezxml_t bbox = ezxml_add_child(layer,"ows:WGS84BoundingBox",0); + ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0), + apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.minx, tileset->wgs84bbox.miny)); + ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0), + apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.maxx, tileset->wgs84bbox.maxy)); + } + + ezxml_set_txt(ezxml_add_child(layer,"ows:Identifier",0),tileset->name); style = ezxml_add_child(layer,"Style",0); @@ -363,8 +372,8 @@ if(tileset->dimensions) { for(i=0; idimensions->nelts; i++) { - const char **values; - const char **value; + apr_array_header_t *values; + int value_idx; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); ezxml_t dim = ezxml_add_child(layer,"Dimension",0); ezxml_set_txt(ezxml_add_child(dim,"ows:Identifier",0),dimension->name); @@ -374,10 +383,9 @@ ezxml_set_txt(ezxml_add_child(dim,"UOM",0),dimension->unit); } values = dimension->print_ogc_formatted_values(ctx,dimension); - value = values; - while(*value) { - ezxml_set_txt(ezxml_add_child(dim,"Value",0),*value); - value++; + for(value_idx=0;value_idxnelts;value_idx++) { + char *idval = APR_ARRAY_IDX(values,value_idx,char*); + ezxml_set_txt(ezxml_add_child(dim,"Value",0),idval); } dimensionstemplate = apr_pstrcat(ctx->pool,dimensionstemplate,"{",dimension->name,"}/",NULL); } @@ -390,13 +398,6 @@ } } - if(tileset->wgs84bbox.minx != tileset->wgs84bbox.maxx) { - ezxml_t bbox = ezxml_add_child(layer,"ows:WGS84BoundingBox",0); - ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0), - apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.minx, tileset->wgs84bbox.miny)); - ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0), - apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.maxx, tileset->wgs84bbox.maxy)); - } for(i=0; igrid_links->nelts; i++) { mapcache_grid_link *grid_link = APR_ARRAY_IDX(tileset->grid_links,i,mapcache_grid_link*); @@ -413,16 +414,17 @@ int j; for(j=0; jgrid->nlevels; j++) { ezxml_t matrixlimits = ezxml_add_child(limits,"TileMatrixLimits",0); + int row; ezxml_set_txt(ezxml_add_child(matrixlimits,"TileMatrix",0), apr_psprintf(ctx->pool,"%s:%d",grid_link->grid->name,j)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MinTileRow",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].minx)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].miny)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MaxTileRow",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxx-1)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxy-1)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MinTileCol",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].miny)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].minx)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MaxTileCol",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxy-1)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxx-1)); } } @@ -701,7 +703,7 @@ continue; } } - if(tileset->timedimension) { + if(!timedim && tileset->timedimension) { timedim = key; continue; } @@ -779,19 +781,20 @@ for(i=0; idimensions->nelts; i++) { char *tmpval; int ok; + const char *value; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - const char *value = apr_table_get(dimtable,dimension->name); + if(dimension->skip_validation) continue; + value = apr_table_get(dimtable,dimension->name); if(value) { tmpval = apr_pstrdup(ctx->pool,value); ok = dimension->validate(ctx,dimension,&tmpval); GC_CHECK_ERROR(ctx); if(ok != MAPCACHE_SUCCESS) { ctx->set_error(ctx,404,"dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); + dimension->name, value); if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","%s",dimension->name); return; } - /* re-set the eventually modified value in the dimension table */ apr_table_set(dimtable,dimension->name,tmpval); } @@ -913,7 +916,7 @@ mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc( ctx->pool,sizeof(mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; if(timedim) { timedim_selected = mapcache_timedimension_get_entries_for_value(ctx, tileset->timedimension, tileset, grid_link->grid, extent, timedim); diff -Nru mapcache-1.2.1/lib/source_mapserver.c mapcache-1.4.1/lib/source_mapserver.c --- mapcache-1.2.1/lib/source_mapserver.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/source_mapserver.c 2016-02-25 15:47:16.000000000 +0000 @@ -278,6 +278,7 @@ mapcache_source* mapcache_source_mapserver_create(mapcache_context *ctx) { ctx->set_error(ctx, 500, "mapserver source not configured for this build"); + return NULL; } #endif diff -Nru mapcache-1.2.1/lib/source_wms.c mapcache-1.4.1/lib/source_wms.c --- mapcache-1.2.1/lib/source_wms.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/source_wms.c 2016-02-25 15:47:16.000000000 +0000 @@ -40,6 +40,7 @@ void _mapcache_source_wms_render_map(mapcache_context *ctx, mapcache_map *map) { mapcache_source_wms *wms = (mapcache_source_wms*)map->tileset->source; + mapcache_http *http; apr_table_t *params = apr_table_clone(ctx->pool,wms->wms_default_params); apr_table_setn(params,"BBOX",apr_psprintf(ctx->pool,"%f,%f,%f,%f", map->extent.minx,map->extent.miny,map->extent.maxx,map->extent.maxy)); @@ -54,7 +55,12 @@ int i; for(i=0; inelts; i++) { apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); + /* set both DIM_key=val and key=val KVP params */ apr_table_setn(params,entry.key,entry.val); + if(strcasecmp(entry.key,"TIME") && strcasecmp(entry.key,"ELEVATION")) { + char *dim_name = apr_pstrcat(ctx->pool,"DIM_",entry.key,NULL); + apr_table_setn(params,dim_name,entry.val); + } } } @@ -68,7 +74,9 @@ } map->encoded_data = mapcache_buffer_create(30000,ctx->pool); - mapcache_http_do_request_with_params(ctx,wms->http,params,map->encoded_data,NULL,NULL); + http = mapcache_http_clone(ctx, wms->http); + http->url = mapcache_http_build_url(ctx,http->url,params); + mapcache_http_do_request(ctx,http,map->encoded_data,NULL,NULL); GC_CHECK_ERROR(ctx); if(!mapcache_imageio_is_valid_format(ctx,map->encoded_data)) { @@ -81,6 +89,7 @@ void _mapcache_source_wms_query(mapcache_context *ctx, mapcache_feature_info *fi) { mapcache_map *map = (mapcache_map*)fi; + mapcache_http *http; mapcache_source_wms *wms = (mapcache_source_wms*)map->tileset->source; apr_table_t *params = apr_table_clone(ctx->pool,wms->wms_default_params); @@ -107,7 +116,9 @@ } fi->data = mapcache_buffer_create(30000,ctx->pool); - mapcache_http_do_request_with_params(ctx,wms->http,params,fi->data,NULL,NULL); + http = mapcache_http_clone(ctx, wms->http); + http->url = mapcache_http_build_url(ctx,http->url,params); + mapcache_http_do_request(ctx,http,fi->data,NULL,NULL); GC_CHECK_ERROR(ctx); } diff -Nru mapcache-1.2.1/lib/strptime.c mapcache-1.4.1/lib/strptime.c --- mapcache-1.2.1/lib/strptime.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/strptime.c 2016-02-25 15:47:16.000000000 +0000 @@ -40,7 +40,6 @@ #include #include "util.h" - static const char *abb_weekdays[] = { "Sun", "Mon", diff -Nru mapcache-1.2.1/lib/tileset.c mapcache-1.4.1/lib/tileset.c --- mapcache-1.2.1/lib/tileset.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/tileset.c 2016-02-25 15:47:16.000000000 +0000 @@ -71,7 +71,7 @@ { /* check we have all we want */ - if(tileset->cache == NULL) { + if(tileset->_cache == NULL) { /* TODO: we should allow tilesets with no caches */ ctx->set_error(ctx, 400, "tileset \"%s\" has no cache configured.", tileset->name); return; @@ -176,7 +176,8 @@ mapcache_grid_link *grid_link, mapcache_extent *bbox, int width, int height, int *ntiles, - mapcache_tile ***tiles) + mapcache_tile ***tiles, + mapcache_grid_link **effectively_used_grid_link) { double resolution; int level; @@ -185,25 +186,25 @@ int x,y; int i=0; resolution = mapcache_grid_get_resolution(bbox, width, height); - mapcache_grid_get_closest_level(ctx,grid_link,resolution,&level); - + *effectively_used_grid_link = mapcache_grid_get_closest_wms_level(ctx,grid_link,resolution,&level); + /* we don't want to assemble tiles that have already been reassembled from a lower level */ - if(grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE && level > grid_link->max_cached_zoom) { - level = grid_link->max_cached_zoom; + if((*effectively_used_grid_link)->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE && level > (*effectively_used_grid_link)->max_cached_zoom) { + level = (*effectively_used_grid_link)->max_cached_zoom; } - mapcache_grid_get_xy(ctx,grid_link->grid,bbox->minx,bbox->miny,level,&bl_x,&bl_y); - mapcache_grid_get_xy(ctx,grid_link->grid,bbox->maxx,bbox->maxy,level,&tr_x,&tr_y); - Mx = MAPCACHE_MAX(tr_x,bl_x); - My = MAPCACHE_MAX(tr_y,bl_y); - mx = MAPCACHE_MIN(tr_x,bl_x); - my = MAPCACHE_MIN(tr_y,bl_y); + mapcache_grid_get_xy(ctx,(*effectively_used_grid_link)->grid,bbox->minx,bbox->miny,level,&bl_x,&bl_y); + mapcache_grid_get_xy(ctx,(*effectively_used_grid_link)->grid,bbox->maxx,bbox->maxy,level,&tr_x,&tr_y); + Mx = MAPCACHE_MAX(MAPCACHE_MIN(MAPCACHE_MAX(tr_x,bl_x),(*effectively_used_grid_link)->grid_limits[level].maxx),(*effectively_used_grid_link)->grid_limits[level].minx); + My = MAPCACHE_MAX(MAPCACHE_MIN(MAPCACHE_MAX(tr_y,bl_y),(*effectively_used_grid_link)->grid_limits[level].maxy),(*effectively_used_grid_link)->grid_limits[level].miny); + mx = MAPCACHE_MIN(MAPCACHE_MAX(MAPCACHE_MIN(tr_x,bl_x),(*effectively_used_grid_link)->grid_limits[level].minx),(*effectively_used_grid_link)->grid_limits[level].maxx); + my = MAPCACHE_MIN(MAPCACHE_MAX(MAPCACHE_MIN(tr_y,bl_y),(*effectively_used_grid_link)->grid_limits[level].miny),(*effectively_used_grid_link)->grid_limits[level].maxy); *ntiles = (Mx-mx+1)*(My-my+1); i=0; *tiles = (mapcache_tile**)apr_pcalloc(ctx->pool, *ntiles*sizeof(mapcache_tile*)); for(x=mx; x<=Mx; x++) { for(y=my; y<=My; y++) { - mapcache_tile *tile = mapcache_tileset_tile_create(ctx->pool,tileset, grid_link); + mapcache_tile *tile = mapcache_tileset_tile_create(ctx->pool,tileset, (*effectively_used_grid_link)); tile->x = x; tile->y = y; tile->z = level; @@ -405,7 +406,7 @@ /* configured metatile size in geographical units */ fullgwidth = res * tileset->metasize_x * grid->tile_sx; fullgheight = res * tileset->metasize_y * grid->tile_sy; - + switch(grid->origin) { case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: mt->map.extent.minx = grid->extent.minx + mt->x * fullgwidth - gbuffer; @@ -459,12 +460,12 @@ GC_CHECK_ERROR(ctx); mapcache_image_metatile_split(ctx, mt); GC_CHECK_ERROR(ctx); - if(mt->map.tileset->cache->tile_multi_set) { - mt->map.tileset->cache->tile_multi_set(ctx, mt->tiles, mt->ntiles); + if(mt->map.tileset->_cache->tile_multi_set) { + mt->map.tileset->_cache->tile_multi_set(ctx, mt->map.tileset->_cache, mt->tiles, mt->ntiles); } else { for(i=0; intiles; i++) { mapcache_tile *tile = &(mt->tiles[i]); - mt->map.tileset->cache->tile_set(ctx, tile); + mt->map.tileset->_cache->tile_set(ctx, mt->map.tileset->_cache, tile); GC_CHECK_ERROR(ctx); } } @@ -504,7 +505,7 @@ dst->grid_links = src->grid_links; dst->config = src->config; dst->name = src->name; - dst->cache = src->cache; + dst->_cache = src->_cache; dst->source = src->source; dst->watermark = src->watermark; dst->wgs84bbox = src->wgs84bbox; @@ -554,6 +555,7 @@ tile->x = src->x; tile->y = src->y; tile->z = src->z; + tile->allow_redirect = src->allow_redirect; return tile; } @@ -616,20 +618,21 @@ } void mapcache_tileset_assemble_out_of_zoom_tile(mapcache_context *ctx, mapcache_tile *tile) { - assert(tile->grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE); - - /* we have at most 4 tiles composing the requested tile */ mapcache_extent tile_bbox; double shrink_x, shrink_y, scalefactor; int x[4],y[4]; int i, n=1; + mapcache_tile *childtile; + assert(tile->grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE); + + /* we have at most 4 tiles composing the requested tile */ mapcache_grid_get_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z, &tile_bbox); /* shrink the extent so we do not fall exactly on a tile boundary, to avoid rounding errors when computing the x,y of the lower level tile(s) we will need */ - + shrink_x = (tile_bbox.maxx - tile_bbox.minx) / (tile->grid_link->grid->tile_sx * 1000); /* 1/1000th of a pixel */ shrink_y = (tile_bbox.maxy - tile_bbox.miny) / (tile->grid_link->grid->tile_sy * 1000); /* 1/1000th of a pixel */ tile_bbox.maxx -= shrink_x; @@ -654,15 +657,15 @@ tile_bbox.minx -= shrink_x; tile_bbox.miny -= shrink_y; - mapcache_tile *childtile = mapcache_tileset_tile_clone(ctx->pool,tile); + childtile = mapcache_tileset_tile_clone(ctx->pool,tile); childtile->z = tile->grid_link->max_cached_zoom; scalefactor = childtile->grid_link->grid->levels[childtile->z]->resolution/tile->grid_link->grid->levels[tile->z]->resolution; tile->nodata = 1; for(i=0;ix = x[i]; - childtile->y = y[i]; mapcache_extent childtile_bbox; double dstminx,dstminy; + childtile->x = x[i]; + childtile->y = y[i]; mapcache_tileset_tile_get(ctx,childtile); GC_CHECK_ERROR(ctx); if(childtile->nodata) { @@ -698,13 +701,14 @@ * https://bugs.freedesktop.org/show_bug.cgi?id=46277 */ unsigned int row,col; unsigned char *srcpixptr; + unsigned char *row_ptr; unsigned int dstminxi = - dstminx / scalefactor; unsigned int dstminyi = - dstminy / scalefactor; srcpixptr = &(childtile->raw_image->data[dstminyi * childtile->raw_image->stride + dstminxi * 4]); /* ctx->log(ctx, MAPCACHE_WARN, "factor: %g. pixel: %d,%d (val:%d)",scalefactor,dstminxi,dstminyi,*((unsigned int*)srcpixptr)); */ - unsigned char *row_ptr = tile->raw_image->data; + row_ptr = tile->raw_image->data; for(row=0;rowraw_image->h;row++) { unsigned char *pix_ptr = row_ptr; for(col=0;colraw_image->w;col++) { @@ -724,8 +728,8 @@ - - + + } void mapcache_tileset_outofzoom_get(mapcache_context *ctx, mapcache_tile *tile) { @@ -761,14 +765,14 @@ */ void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile) { - int isLocked,ret; + int ret; mapcache_metatile *mt=NULL; if(tile->grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && tile->z > tile->grid_link->max_cached_zoom) { mapcache_tileset_outofzoom_get(ctx, tile); return; } - ret = tile->tileset->cache->tile_get(ctx, tile); + ret = tile->tileset->_cache->tile_get(ctx, tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(ret == MAPCACHE_SUCCESS && tile->tileset->auto_expire && tile->mtime && tile->tileset->source && !tile->tileset->read_only) { @@ -778,14 +782,12 @@ apr_time_t now = apr_time_now(); apr_time_t stale = tile->mtime + apr_time_from_sec(tile->tileset->auto_expire); if(staletileset->read_only || !tile->tileset->source) { /* there is no source configured for this tile. not an error, let caller now*/ @@ -802,45 +804,71 @@ ctx->set_error(ctx,404,"tile not in cache, and configured for readonly mode"); return; } + } - /* the tile does not exist, we must take action before re-asking for it */ - /* - * is the tile already being rendered by another thread ? - * the call is protected by the same mutex that sets the lock on the tile, - * so we can assure that: - * - if the lock does not exist, then this thread should do the rendering - * - if the lock exists, we should wait for the other thread to finish - */ - /* aquire a lock on the metatile */ - mt = mapcache_tileset_metatile_get(ctx, tile); - isLocked = mapcache_lock_or_wait_for_resource(ctx, mapcache_tileset_metatile_resource_key(ctx,mt)); + if (ret == MAPCACHE_CACHE_MISS || ret == MAPCACHE_CACHE_RELOAD) { + int isLocked; + void *lock; + /* If the tile does not exist or stale, we must take action before re-asking for it */ + if( !tile->tileset->read_only && tile->tileset->source && !ctx->config->non_blocking) { + /* + * is the tile already being rendered by another thread ? + * the call is protected by the same mutex that sets the lock on the tile, + * so we can assure that: + * - if the lock does not exist, then this thread should do the rendering + * - if the lock exists, we should wait for the other thread to finish + */ + + /* aquire a lock on the metatile */ + mt = mapcache_tileset_metatile_get(ctx, tile); + isLocked = mapcache_lock_or_wait_for_resource(ctx, ctx->config->locker, mapcache_tileset_metatile_resource_key(ctx,mt), &lock); + GC_CHECK_ERROR(ctx); - if(isLocked == MAPCACHE_TRUE) { - /* no other thread is doing the rendering, do it ourselves */ + + if(isLocked == MAPCACHE_TRUE) { + /* no other thread is doing the rendering, do it ourselves */ #ifdef DEBUG - ctx->log(ctx, MAPCACHE_DEBUG, "cache miss: tileset %s - tile %d %d %d", - tile->tileset->name,tile->x, tile->y,tile->z); + ctx->log(ctx, MAPCACHE_DEBUG, "cache miss/reload: tileset %s - tile %d %d %d", + tile->tileset->name,tile->x, tile->y,tile->z); #endif - /* this will query the source to create the tiles, and save them to the cache */ - mapcache_tileset_render_metatile(ctx, mt); + /* this will query the source to create the tiles, and save them to the cache */ + mapcache_tileset_render_metatile(ctx, mt); - mapcache_unlock_resource(ctx, mapcache_tileset_metatile_resource_key(ctx,mt)); + if(GC_HAS_ERROR(ctx)) { + /* temporarily clear error state so we don't mess up with error handling in the locker */ + void *error; + ctx->pop_errors(ctx,&error); + mapcache_unlock_resource(ctx, ctx->config->locker, mapcache_tileset_metatile_resource_key(ctx,mt), lock); + ctx->push_errors(ctx,error); + } else { + mapcache_unlock_resource(ctx, ctx->config->locker, mapcache_tileset_metatile_resource_key(ctx,mt), lock); + } + } } - GC_CHECK_ERROR(ctx); - /* the previous step has successfully finished, we can now query the cache to return the tile content */ - ret = tile->tileset->cache->tile_get(ctx, tile); - GC_CHECK_ERROR(ctx); + if (ret == MAPCACHE_CACHE_RELOAD && GC_HAS_ERROR(ctx)) + /* If we tried to reload a stale tile but failed, we know we have already + * fetched it from the cache. We can then ignore errors and just use old tile. + */ + ctx->clear_errors(ctx); - if(ret != MAPCACHE_SUCCESS) { - if(isLocked == MAPCACHE_FALSE) { - ctx->set_error(ctx, 500, "tileset %s: unknown error (another thread/process failed to create the tile I was waiting for)", - tile->tileset->name); - } else { - /* shouldn't really happen, as the error ought to have been caught beforehand */ - ctx->set_error(ctx, 500, "tileset %s: failed to re-get tile %d %d %d from cache after set", tile->tileset->name,tile->x,tile->y,tile->z); + else { + /* Else, check for errors and try to fetch the tile from the cache. + */ + GC_CHECK_ERROR(ctx); + ret = tile->tileset->_cache->tile_get(ctx, tile->tileset->_cache, tile); + GC_CHECK_ERROR(ctx); + + if(ret != MAPCACHE_SUCCESS) { + if(isLocked == MAPCACHE_FALSE) { + ctx->set_error(ctx, 500, "tileset %s: unknown error (another thread/process failed to create the tile I was waiting for)", + tile->tileset->name); + } else { + /* shouldn't really happen, as the error ought to have been caught beforehand */ + ctx->set_error(ctx, 500, "tileset %s: failed to re-get tile %d %d %d from cache after set", tile->tileset->name,tile->x,tile->y,tile->z); + } } } } @@ -856,7 +884,7 @@ { int i; /*delete the tile itself*/ - tile->tileset->cache->tile_delete(ctx,tile); + tile->tileset->_cache->tile_delete(ctx,tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(whole_metatile) { @@ -865,7 +893,7 @@ mapcache_tile *subtile = &mt->tiles[i]; /* skip deleting the actual tile */ if(subtile->x == tile->x && subtile->y == tile->y) continue; - subtile->tileset->cache->tile_delete(ctx,subtile); + subtile->tileset->_cache->tile_delete(ctx,subtile->tileset->_cache,subtile); /* silently pass failure if the tile was not found */ if(ctx->get_error(ctx) == 404) { ctx->clear_errors(ctx); diff -Nru mapcache-1.2.1/lib/util.c mapcache-1.4.1/lib/util.c --- mapcache-1.2.1/lib/util.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/lib/util.c 2016-02-25 15:47:16.000000000 +0000 @@ -33,6 +33,7 @@ #include #include #include +#include #ifndef _WIN32 #include @@ -41,8 +42,63 @@ #ifndef M_PI #define M_PI 3.14159265358979323846264338327 #endif + +#ifdef _WIN32 +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long int uint64_t; +#endif + const double mapcache_meters_per_unit[MAPCACHE_UNITS_COUNT] = {1.0,6378137.0 * 2.0 * M_PI / 360,0.3048}; + +static char encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/'}; +static int mod_table[] = {0, 2, 1}; + + +char *base64_encode(apr_pool_t *pool, const unsigned char *data, size_t input_length) { + int i,j; + char *encoded_data; + size_t output_length = 4 * ((input_length + 2) / 3) + 1; + + encoded_data = (char*)apr_pcalloc(pool,output_length*sizeof(char)); + if (encoded_data == NULL) return NULL; + + for (i = 0, j = 0; i < input_length;) { + + uint32_t octet_a; + uint32_t octet_b; + uint32_t octet_c; + uint32_t triple; + octet_a = i < input_length ? (unsigned char)data[i++] : 0; + octet_b = i < input_length ? (unsigned char)data[i++] : 0; + octet_c = i < input_length ? (unsigned char)data[i++] : 0; + + triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + + encoded_data[j++] = encoding_table[(triple >> 3 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 2 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 1 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 0 * 6) & 0x3F]; + } + + for (i = 0; i < mod_table[input_length % 3]; i++) + encoded_data[output_length - 2 - i] = '='; + + encoded_data[output_length-1]=0; + + return encoded_data; +} + + int mapcache_util_extract_int_list(mapcache_context *ctx, const char* cargs, const char *sdelim, int **numbers, int *numbers_count) { @@ -172,18 +228,18 @@ void _mapcache_context_set_error_default(mapcache_context *ctx, int code, char *msg, ...) { - char *fmt; + char *new_msg; va_list args; va_start(args,msg); + new_msg = apr_pvsprintf(ctx->pool,msg,args); + va_end(args); if(ctx->_errmsg) { - fmt=apr_psprintf(ctx->pool,"%s\n%s",ctx->_errmsg,msg); + ctx->_errmsg = apr_pstrcat(ctx->pool, ctx->_errmsg, "\n", new_msg, NULL); } else { - fmt=msg; + ctx->_errmsg = new_msg; ctx->_errcode = code; } - ctx->_errmsg = apr_pvsprintf(ctx->pool,fmt,args); - va_end(args); } void _mapcache_context_clear_error_default(mapcache_context *ctx) @@ -196,6 +252,47 @@ } +struct _error_log { + int _errcode; + char *_errmsg; + apr_table_t *exceptions; +}; + +void _mapcache_context_pop_errors(mapcache_context *ctx, void **error) +{ + struct _error_log *e = (struct _error_log*)apr_pcalloc(ctx->pool, sizeof(struct _error_log)); + e->_errcode = ctx->_errcode; + e->_errmsg = ctx->_errmsg; + e->exceptions = ctx->exceptions; + ctx->_errcode = 0; + ctx->_errmsg = NULL; + ctx->exceptions = NULL; + *error = e; +} + + +void _mapcache_context_push_errors(mapcache_context *ctx, void *error) +{ + struct _error_log *e = (struct _error_log*)error; + if(e->_errcode) + ctx->_errcode = e->_errcode; + if(e->_errmsg) { + if(ctx->_errmsg) { + ctx->_errmsg = apr_psprintf(ctx->pool,"%s\n%s",e->_errmsg,ctx->_errmsg); + } else { + ctx->_errmsg = e->_errmsg; + } + } + if(e->exceptions) { + if(ctx->exceptions) { + apr_table_overlap(ctx->exceptions, e->exceptions, APR_OVERLAP_TABLES_SET); + } else { + ctx->exceptions = e->exceptions; + } + } +} + + void mapcache_context_init(mapcache_context *ctx) { ctx->_errcode = 0; @@ -205,6 +302,9 @@ ctx->set_error = _mapcache_context_set_error_default; ctx->set_exception = _mapcache_context_set_exception_default; ctx->clear_errors = _mapcache_context_clear_error_default; + ctx->pop_errors = _mapcache_context_pop_errors; + ctx->push_errors = _mapcache_context_push_errors; + ctx->headers_in = NULL; } void mapcache_context_copy(mapcache_context *src, mapcache_context *dst) @@ -225,7 +325,11 @@ dst->service = src->service; dst->exceptions = src->exceptions; dst->threadlock = src->threadlock; - dst->process_pool = src->process_pool; + dst->supports_redirects = src->supports_redirects; + dst->pop_errors = src->pop_errors; + dst->push_errors = src->push_errors; + dst->connection_pool = src->connection_pool; + dst->headers_in = src->headers_in; } char* mapcache_util_get_tile_dimkey(mapcache_context *ctx, mapcache_tile *tile, char* sanitized_chars, char *sanitize_to) @@ -297,9 +401,41 @@ return path; } - -/* vim: ts=2 sts=2 et sw=2 -*/ +void mapcache_make_parent_dirs(mapcache_context *ctx, char *filename) { + char *hackptr1,*hackptr2=NULL; + apr_status_t ret; + char errmsg[120]; + + /* find the location of the last '/' in the string */ + hackptr1 = filename; + while(*hackptr1) { + if(*hackptr1 == '/') + hackptr2 = hackptr1; + hackptr1++; + } + + if(hackptr2) { + /* terminate string on last '/' */ + *hackptr2 = '\0'; + } + + ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool); + + if(hackptr2) { + *hackptr2 = '/'; + } + + + if(APR_SUCCESS != ret) { + /* + * apr_dir_make_recursive sometimes sends back this error, although it should not. + * ignore this one + */ + if(!APR_STATUS_IS_EEXIST(ret)) { + ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); + } + } +} #if defined(_WIN32) && !defined(__CYGWIN__) @@ -353,3 +489,8 @@ #endif + +/* vim: ts=2 sts=2 et sw=2 +*/ + + diff -Nru mapcache-1.2.1/mapcache.xml mapcache-1.4.1/mapcache.xml --- mapcache-1.2.1/mapcache.xml 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/mapcache.xml 2016-02-25 15:47:16.000000000 +0000 @@ -7,6 +7,10 @@ /tmp + + /tmp/{tileset}-{z}-{grid}.db + + @@ -23,9 +27,9 @@ vmap0 - disk + sqlite WGS84 - g + GoogleMapsCompatible PNG 5 5 10 @@ -49,6 +53,9 @@ report - /tmp + + /tmp + 300 + diff -Nru mapcache-1.2.1/mapcache.xml.sample mapcache-1.4.1/mapcache.xml.sample --- mapcache-1.2.1/mapcache.xml.sample 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/mapcache.xml.sample 2016-02-25 15:47:16.000000000 +0000 @@ -73,6 +73,16 @@ + + value + + + create table if not exists tiles(tileset text, grid text, x integer, y integer, z integer, data blob, dim text, ctime datetime, primary key(tileset,grid,x,y,z,dim)) + select 1 from tiles where x=:x and y=:y and z=:z and dim=:dim and tileset=:tileset and grid=:grid + select data,strftime("%s",ctime) from tiles where tileset=:tileset and grid=:grid and x=:x and y=:y and z=:z and dim=:dim + insert or replace into tiles(tileset,grid,x,y,z,data,dim,ctime) values (:tileset,:grid,:x,:y,:z,:data,:dim,datetime('now')) + delete from tiles where x=:x and y=:y and z=:z and dim=:dim and tileset=:tileset and grid=:grid + {tileset}-{grid}-{dim}-{z}-{y}-{x}.{ext} + + + https://myserver/webdav/{tileset}/{grid}/{z}/{x}/{y}.{ext} + + my-virtualhost-alias.domain.com + + + + foo + + + + + foo + + + + + foo + + + + + foo + + + + + https://foo.s3.amazonaws.com/tiles/{tileset}/{grid}/{z}/{x}/{y}/{ext} + + foo.s3.amazonaws.com + + foo + foo/sdsvd + eu-west-1 + + + REDUCED_REDUNDANCY + public-read + + + + + https://foo.blob.core.windows.net/tiles/{tileset}/{grid}/{z}/{x}/{y}/{ext} + + foo.blob.core.windows.net + + foo + foobarcccccccccccccccccccccyA== + tiles + + + + https://storage.googleapis.com/mytiles-mapcache/{tileset}/{grid}/{z}/{x}/{y}.{ext} + GOOGPGDWFDG345SDFGSD + sdfgsdSDFwedfwefr2345324dfsGdsfgs + + + public-read + + + + - /tmp + /tmp + 0.01 + + + /tmp + + + + + localhost + 11211 + + + memcache-host + 11212 + + 0.3 + + + true diff -Nru mapcache-1.2.1/nginx/ngx_http_mapcache_module.c mapcache-1.4.1/nginx/ngx_http_mapcache_module.c --- mapcache-1.2.1/nginx/ngx_http_mapcache_module.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/nginx/ngx_http_mapcache_module.c 2016-02-25 15:47:16.000000000 +0000 @@ -62,7 +62,7 @@ apr_pool_create(&process_pool,NULL); mapcache_context *ctx = apr_pcalloc(process_pool, sizeof(mapcache_ngx_context)); ctx->pool = process_pool; - ctx->process_pool = process_pool; + ctx->connection_pool = NULL; ctx->threadlock = NULL; mapcache_context_init(ctx); ctx->log = ngx_mapcache_context_log; @@ -211,7 +211,6 @@ mapcache_ngx_context *ngctx = ngx_http_get_module_loc_conf(r, ngx_http_mapcache_module); mapcache_context *ctx = (mapcache_context*)ngctx; apr_pool_create(&(ctx->pool),process_pool); - ctx->process_pool = process_pool; ngctx->r = r; mapcache_request *request = NULL; mapcache_http_response *http_response; @@ -298,6 +297,7 @@ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,ctx->get_error_message(ctx)); return NGX_CONF_ERROR; } + mapcache_connection_pool_create(&ctx->connection_pool,ctx->pool); ctx->config->non_blocking = 1; ngx_http_core_loc_conf_t *clcf; diff -Nru mapcache-1.2.1/release.sh mapcache-1.4.1/release.sh --- mapcache-1.2.1/release.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/release.sh 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,54 @@ +#!/bin/bash + +ms_version=$1 + +echo "release steps for mapcache version $ms_version (mapcache-$ms_version.tar.gz)" +echo "" + +ms_version_suffix=`echo $ms_version | cut -s -d- -f2` +if [ -z $ms_version_suffix ]; then + ms_version_num=$ms_version +else + ms_version_num=`echo $ms_version | cut -s -d- -f1` +fi + +ms_version_major=`echo $ms_version_num | cut -d. -f1` +ms_version_minor=`echo $ms_version_num | cut -d. -f2` +ms_version_revision=`echo $ms_version_num | cut -d. -f3` + + +tagname=rel-$ms_version_major-$ms_version_minor-$ms_version_revision +if [ ! -z $ms_version_suffix ]; then + tagname=$tagname-$ms_version_suffix +fi + +echo "#" +echo "# make sure:" +echo "# - you are on branch-$ms_version_major-$ms_version_minor" +echo "# - you have edited HISTORY.TXT with changes related to this release" +echo "#" +echo "" + +echo "sed -i '/set (MAPCACHE_VERSION_MAJOR/c\set (MAPCACHE_VERSION_MAJOR $ms_version_major)' CMakeLists.txt" +echo "sed -i '/set (MAPCACHE_VERSION_MINOR/c\set (MAPCACHE_VERSION_MINOR $ms_version_minor)' CMakeLists.txt" +echo "sed -i '/set (MAPCACHE_VERSION_REVISION/c\set (MAPCACHE_VERSION_REVISION $ms_version_revision)' CMakeLists.txt" +if [ ! -z $ms_version_suffix ]; then + echo "sed -i '/set (MAPCACHE_VERSION_SUFFIX/c\set (MAPCACHE_VERSION_SUFFIX \"-$ms_version_suffix\")' CMakeLists.txt" +else + echo "sed -i '/set (MAPCACHE_VERSION_SUFFIX/c\set (MAPCACHE_VERSION_SUFFIX \"\")' CMakeLists.txt" +fi + +echo "git add CMakeLists.txt" +echo "git commit -m \"update for $ms_version release\"" +echo "git tag -a $tagname -m \"Create $ms_version tag\"" +echo "git push origin branch-$ms_version_major-$ms_version_minor --tags" +echo "git archive --format=tar.gz --prefix=mapcache-$ms_version/ $tagname >/tmp/mapcache-$ms_version.tar.gz" +echo "scp /tmp/mapcache-$ms_version.tar.gz download.osgeo.org:/osgeo/download/mapserver" + +echo "" +echo "#" +echo "#optionally update doc site, these commands need tweaking before being ran" +echo "#" + +echo "/path/to/docs/scripts/changelog.sh rel-previous-tag..$tagname >> /path/to/docs/en/development/changelog/changelog-$ms_version_major-$ms_version_minor.txt" + diff -Nru mapcache-1.2.1/scripts/vagrant/mapcache.sh mapcache-1.4.1/scripts/vagrant/mapcache.sh --- mapcache-1.2.1/scripts/vagrant/mapcache.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/scripts/vagrant/mapcache.sh 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,13 @@ +#!/bin/sh + +NUMTHREADS=2 # we have 2 cpus configured +export NUMTHREADS + +cd /vagrant + +mkdir build_vagrant +cd build_vagrant +cmake -DWITH_MEMCACHE=1 .. + +make -j $NUMTHREADS +make install diff -Nru mapcache-1.2.1/scripts/vagrant/packages.sh mapcache-1.4.1/scripts/vagrant/packages.sh --- mapcache-1.2.1/scripts/vagrant/packages.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/scripts/vagrant/packages.sh 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,17 @@ +#!/bin/sh + +sed -i 's#deb http://us.archive.ubuntu.com/ubuntu/#deb mirror://mirrors.ubuntu.com/mirrors.txt#' /etc/apt/sources.list + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y python-software-properties +add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable +apt-get update +apt-get -y upgrade + +# install packages we need +apt-get install -q -y build-essential pkg-config cmake libgeos-dev rake vim \ + bison flex libgdal1-dev libproj-dev libpng12-dev libjpeg-dev libfcgi-dev \ + libcurl4-gnutls-dev apache2-prefork-dev libtiff4-dev libpixman-1-dev \ + libsqlite3-dev libmemcached-dev diff -Nru mapcache-1.2.1/scripts/vagrant/virtualbox-fix.sh mapcache-1.4.1/scripts/vagrant/virtualbox-fix.sh --- mapcache-1.2.1/scripts/vagrant/virtualbox-fix.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/scripts/vagrant/virtualbox-fix.sh 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ ! -e "/usr/lib/VBoxGuestAdditions" ]; then + ln -s /opt/VBoxGuestAdditions-4.3.10/lib/VBoxGuestAdditions /usr/lib/VBoxGuestAdditions +fi diff -Nru mapcache-1.2.1/util/mapcache_seed.c mapcache-1.4.1/util/mapcache_seed.c --- mapcache-1.2.1/util/mapcache_seed.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.4.1/util/mapcache_seed.c 2016-02-25 15:47:16.000000000 +0000 @@ -37,6 +37,7 @@ #include #ifndef _WIN32 #include +#include #define USE_FORK #include #endif @@ -53,6 +54,7 @@ #include apr_queue_t *work_queue; +apr_queue_t *log_queue; #if defined(USE_OGR) && defined(USE_GEOS) #define USE_CLIPPERS @@ -80,10 +82,13 @@ int force = 0; int sig_int_received = 0; int error_detected = 0; +double percent_failed_allowed = 1.0; +int n_metatiles_tot=0; +FILE *failed_log = NULL, *retry_log = NULL; +#define FAIL_BACKLOG_COUNT 1000 apr_time_t age_limit = 0; -int seededtilestot=0, seededtiles=0, queuedtilestot=0; -struct mctimeval lastlogtime,starttime; +struct mctimeval starttime; typedef enum { MAPCACHE_CMD_SEED, @@ -96,7 +101,8 @@ typedef enum { MAPCACHE_ITERATION_UNSET, MAPCACHE_ITERATION_DEPTH_FIRST, - MAPCACHE_ITERATION_LEVEL_FIRST + MAPCACHE_ITERATION_LEVEL_FIRST, + MAPCACHE_ITERATION_LOG } mapcache_iteration_mode; mapcache_iteration_mode iteration_mode = MAPCACHE_ITERATION_UNSET; @@ -108,6 +114,18 @@ int z; }; +typedef enum { + MAPCACHE_STATUS_OK, + MAPCACHE_STATUS_FAIL, + MAPCACHE_STATUS_FINISHED +} s_status; + +struct seed_status { + s_status status; + int x,y,z; + char *msg; +}; + #ifdef USE_FORK struct msg_cmd { long mtype; @@ -119,6 +137,7 @@ int push_queue(struct seed_cmd cmd) { + struct seed_cmd *pcmd; #ifdef USE_FORK if(nprocesses > 1) { struct msg_cmd mcmd; @@ -131,7 +150,7 @@ return APR_SUCCESS; } #endif - struct seed_cmd *pcmd = calloc(1,sizeof(struct seed_cmd)); + pcmd = calloc(1,sizeof(struct seed_cmd)); *pcmd = cmd; return apr_queue_push(work_queue,pcmd); } @@ -152,7 +171,7 @@ return APR_SUCCESS; } #endif - + ret = apr_queue_pop(work_queue, (void**)&pcmd); if(ret == APR_SUCCESS) { *cmd = *pcmd; @@ -191,6 +210,7 @@ static const apr_getopt_option_t seed_options[] = { /* long-option, short-option, has-arg flag, description */ { "config", 'c', TRUE, "configuration file (/path/to/mapcache.xml)"}, + { "cache", 'C', TRUE, "override cache used by selected tileset (useful for selectively seeding fallback/multitier caches)"}, #ifdef USE_CLIPPERS { "ogr-datasource", 'd', TRUE, "ogr datasource to get features from"}, #endif @@ -203,12 +223,15 @@ #ifdef USE_CLIPPERS { "ogr-layer", 'l', TRUE, "layer inside datasource"}, #endif + { "log-failed", 'L', TRUE, "log failed tiles to [file]"}, { "mode", 'm', TRUE, "mode: seed (default), delete or transfer" }, { "metasize", 'M', TRUE, "override metatile size while seeding, eg 8,8" }, { "nthreads", 'n', TRUE, "number of parallel threads to use (incompatible with -p/--nprocesses)" }, { "older", 'o', TRUE, "reseed tiles older than supplied date (format: year/month/day hour:minute, eg: 2011/01/31 20:45" }, { "nprocesses", 'p', TRUE, "number of parallel processes to use (incompatible with -n/--nthreads)" }, + { "percent", 'P', TRUE, "percent of failed requests allowed from the last 1000 before we abort (default: 1(%), set to 0 to abort on first error)" }, { "quiet", 'q', FALSE, "don't show progress info" }, + { "retry-failed", 'R', TRUE, "retry failed requests logged to [file] by --log-failed" }, #ifdef USE_CLIPPERS { "ogr-sql", 's', TRUE, "sql to filter inside layer"}, #endif @@ -259,6 +282,10 @@ { mapcache_metatile *mt = mapcache_tileset_metatile_get(ctx,tile); GEOSCoordSequence *mtbboxls = GEOSCoordSeq_create(5,2); + GEOSGeometry *mtbbox = GEOSGeom_createLinearRing(mtbboxls); + GEOSGeometry *mtbboxg = GEOSGeom_createPolygon(mtbbox,NULL,0); + int i; + int intersects = 0; GEOSCoordSeq_setX(mtbboxls,0,mt->map.extent.minx); GEOSCoordSeq_setY(mtbboxls,0,mt->map.extent.miny); GEOSCoordSeq_setX(mtbboxls,1,mt->map.extent.maxx); @@ -269,10 +296,6 @@ GEOSCoordSeq_setY(mtbboxls,3,mt->map.extent.maxy); GEOSCoordSeq_setX(mtbboxls,4,mt->map.extent.minx); GEOSCoordSeq_setY(mtbboxls,4,mt->map.extent.miny); - GEOSGeometry *mtbbox = GEOSGeom_createLinearRing(mtbboxls); - GEOSGeometry *mtbboxg = GEOSGeom_createPolygon(mtbbox,NULL,0); - int i; - int intersects = 0; for(i=0; i= 1) nworkers = nprocesses; - - sprintf(msg,"seeding tile %d %d %d",x,y,z); - if(lastmsglen) { - char erasestring[1024]; - int len = MAPCACHE_MIN(1023,lastmsglen); - memset(erasestring,' ',len); - erasestring[len+1]='\0'; - sprintf(erasestring,"\r%%%ds\r",lastmsglen); - printf(erasestring," "); - } - lastmsglen = strlen(msg); - printf("%s",msg); - fflush(NULL); - return; - - if(queuedtilestot>nworkers) { - struct mctimeval now_t; - float duration; - float totalduration; - seededtilestot = queuedtilestot - nworkers; - - mapcache_gettimeofday(&now_t,NULL); - duration = ((now_t.tv_sec-lastlogtime.tv_sec)*1000000+(now_t.tv_usec-lastlogtime.tv_usec))/1000000.0; - totalduration = ((now_t.tv_sec-starttime.tv_sec)*1000000+(now_t.tv_usec-starttime.tv_usec))/1000000.0; - if(duration>=5) { - int Nx, Ny, Ntot, Ncur, ntilessincelast; - Nx = (grid_link->grid_limits[z].maxx-grid_link->grid_limits[z].minx)/tileset->metasize_x; - Ny = (grid_link->grid_limits[z].maxy-grid_link->grid_limits[z].miny)/tileset->metasize_y; - Ntot = Nx*Ny; - Ncur = ((y-grid_link->grid_limits[z].miny)/tileset->metasize_y ) * Nx + - (x-grid_link->grid_limits[z].minx+1)/tileset->metasize_x; - ntilessincelast = seededtilestot-seededtiles; - sprintf(msg,"seeding level %d [%d/%d]: %f metatiles/sec (avg since start: %f)",z,Ncur,Ntot,ntilessincelast/duration, - seededtilestot/totalduration); - lastlogtime=now_t; - seededtiles=seededtilestot; - } else { - return; - } - } else { - sprintf(msg,"seeding level %d",z); - - } - if(lastmsglen) { - char erasestring[1024]; - int len = MAPCACHE_MIN(1023,lastmsglen); - memset(erasestring,' ',len); - erasestring[len+1]='\0'; - sprintf(erasestring,"\r%%%ds\r",lastmsglen); - printf(erasestring," "); - } - lastmsglen = strlen(msg); - printf("%s",msg); - fflush(NULL); -} - cmd examine_tile(mapcache_context *ctx, mapcache_tile *tile) { int action = MAPCACHE_CMD_SKIP; - int intersects = -1; - int tile_exists = force?0:tileset->cache->tile_exists(ctx,tile); + int tile_exists; + +#ifdef USE_CLIPPERS + /* check we are in the requested features before checking the tile */ + if(nClippers > 0 && ogr_features_intersect_tile(ctx,tile) == 0) + return MAPCACHE_CMD_SKIP; +#endif + + tile_exists = force?0:tileset->_cache->tile_exists(ctx,tileset->_cache,tile); /* if the tile exists and a time limit was specified, check the tile modification date */ if(tile_exists) { if(age_limit) { - if(tileset->cache->tile_get(ctx,tile) == MAPCACHE_SUCCESS) { + if(tileset->_cache->tile_get(ctx,tileset->_cache, tile) == MAPCACHE_SUCCESS) { if(tile->mtime && tile->mtime 0) { - intersects = ogr_features_intersect_tile(ctx,tile); - } -#endif - if(intersects != 0) { - /* the tile intersects the ogr features, or there was no clipping asked for: seed it */ - if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { - mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); - /* if we are in mode transfer, delete it from the dst tileset */ - if (mode == MAPCACHE_CMD_TRANSFER) { - tile->tileset = tileset_transfer; - if (tileset_transfer->cache->tile_exists(ctx,tile)) { - mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); - } - tile->tileset = tileset; - } - action = mode; - } else { //if(action == MAPCACHE_CMD_DELETE) - action = MAPCACHE_CMD_DELETE; - } - } else { - /* the tile does not intersect the ogr features, and already exists, do nothing */ - action = MAPCACHE_CMD_SKIP; + if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { + mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); + /* if we are in mode transfer, delete it from the dst tileset */ + if (mode == MAPCACHE_CMD_TRANSFER) { + tile->tileset = tileset_transfer; + if (tile->tileset->_cache->tile_exists(ctx,tile->tileset->_cache, tile)) { + mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); } + tile->tileset = tileset; + } + action = mode; + } else { //if(action == MAPCACHE_CMD_DELETE) + action = MAPCACHE_CMD_DELETE; + } } } else { //BUG: tile_exists returned true, but tile_get returned a failure. not sure what to do. @@ -401,7 +355,7 @@ /* the tile exists in the source tileset, check if the tile exists in the destination cache */ tile->tileset = tileset_transfer; - if (tileset_transfer->cache->tile_exists(ctx,tile)) { + if (tile->tileset->_cache->tile_exists(ctx,tile->tileset->_cache, tile)) { action = MAPCACHE_CMD_SKIP; } else { action = MAPCACHE_CMD_TRANSFER; @@ -415,20 +369,7 @@ } else { // the tile does not exist if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { -#ifdef USE_CLIPPERS - /* check we are in the requested features before deleting the tile */ - if(nClippers > 0) { - if(ogr_features_intersect_tile(ctx,tile)) { - action = mode; - } else { - action = MAPCACHE_CMD_SKIP; - } - } else { - action = mode; - } -#else action = mode; -#endif } else { action = MAPCACHE_CMD_SKIP; } @@ -450,9 +391,7 @@ if(sig_int_received || error_detected) { //stop if we were asked to stop by hitting ctrl-c //remove all items from the queue struct seed_cmd entry; - while (trypop_queue(&entry)!=APR_EAGAIN) { - queuedtilestot--; - } + while (trypop_queue(&entry)!=APR_EAGAIN) /*do nothing*/; return; } @@ -466,8 +405,6 @@ cmd.z = tile->z; cmd.command = action; push_queue(cmd); - queuedtilestot++; - progresslog(tile->x,tile->y,tile->z); } //recurse into our 4 child metatiles @@ -559,11 +496,16 @@ if(sig_int_received || error_detected) { //stop if we were asked to stop by hitting ctrl-c //remove all items from the queue struct seed_cmd entry; - while (trypop_queue(&entry)!=APR_EAGAIN) { - queuedtilestot--; - } + while (trypop_queue(&entry)!=APR_EAGAIN) /* do nothing */; break; } + if(iteration_mode == MAPCACHE_ITERATION_LOG) { + if(3 != fscanf(retry_log,"%d,%d,%d\n",&x,&y,&z)) { + break; + } else { + printf("from log: %d %d %d\n",x,y,z); + } + } tile->x = x; tile->y = y; tile->z = z; @@ -577,8 +519,6 @@ cmd.z = z; cmd.command = action; push_queue(cmd); - queuedtilestot++; - progresslog(x,y,z); } //compute next x,y,z @@ -603,10 +543,6 @@ cmd.command = MAPCACHE_CMD_STOP; push_queue(cmd); } - - if(error_detected && ctx.get_error_message(&ctx)) { - printf("%s\n",ctx.get_error_message(&ctx)); - } } @@ -633,27 +569,55 @@ if(cmd.command == MAPCACHE_CMD_SEED) { /* aquire a lock on the metatile ?*/ mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); - int isLocked = mapcache_lock_or_wait_for_resource(&seed_ctx, mapcache_tileset_metatile_resource_key(&seed_ctx,mt)); + void *lock; + int isLocked = mapcache_lock_or_wait_for_resource(&seed_ctx, seed_ctx.config->locker, mapcache_tileset_metatile_resource_key(&seed_ctx,mt), &lock); if(isLocked == MAPCACHE_TRUE) { /* this will query the source to create the tiles, and save them to the cache */ mapcache_tileset_render_metatile(&seed_ctx, mt); - mapcache_unlock_resource(&seed_ctx, mapcache_tileset_metatile_resource_key(&seed_ctx,mt)); + if(GC_HAS_ERROR(&seed_ctx)) { + /* temporarily clear error state so we don't mess up with error handling in the locker */ + void *error; + seed_ctx.pop_errors(&seed_ctx,&error); + mapcache_unlock_resource(&seed_ctx, seed_ctx.config->locker, mapcache_tileset_metatile_resource_key(&seed_ctx,mt), lock); + seed_ctx.push_errors(&seed_ctx,error); + } else { + mapcache_unlock_resource(&seed_ctx, seed_ctx.config->locker, mapcache_tileset_metatile_resource_key(&seed_ctx,mt), lock); + } } } else if (cmd.command == MAPCACHE_CMD_TRANSFER) { int i; mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); for (i = 0; i < mt->ntiles; i++) { mapcache_tile *subtile = &mt->tiles[i]; - mapcache_tileset_tile_get(&seed_ctx, subtile); - subtile->tileset = tileset_transfer; - tileset_transfer->cache->tile_set(&seed_ctx, subtile); + int cache_ret; + cache_ret = tileset->_cache->tile_get(&seed_ctx, tileset->_cache, subtile); + if(cache_ret == MAPCACHE_SUCCESS && !GC_HAS_ERROR(&seed_ctx)) { + subtile->tileset = tileset_transfer; + subtile->tileset->_cache->tile_set(&seed_ctx, subtile->tileset->_cache, subtile); + } } } else { //CMD_DELETE mapcache_tileset_tile_delete(&seed_ctx,tile,MAPCACHE_TRUE); } - if(seed_ctx.get_error(&seed_ctx)) { - error_detected++; - ctx.log(&ctx,MAPCACHE_INFO,seed_ctx.get_error_message(&seed_ctx)); + + { + struct seed_status *st = calloc(1,sizeof(struct seed_status)); + st->x=tile->x; + st->y=tile->y; + st->z=tile->z; + if(seed_ctx.get_error(&seed_ctx)) { + st->status = MAPCACHE_STATUS_FAIL; + st->msg = strdup(seed_ctx.get_error_message(&seed_ctx)); + seed_ctx.clear_errors(&seed_ctx); + } else { + st->status = MAPCACHE_STATUS_OK; + } + ret = apr_queue_push(log_queue,(void*)st); + if(ret != APR_SUCCESS) + { + printf("FATAL ERROR: unable to log progress\n"); + break; + } } } } @@ -669,6 +633,66 @@ return NULL; } +static void* APR_THREAD_FUNC log_thread_fn(apr_thread_t *thread, void *data) { + size_t cur; + double last_time; + double now_time; + int i; + int nfailed; + int ntotal; + double pct; + char failed[FAIL_BACKLOG_COUNT]; + memset(failed,-1,FAIL_BACKLOG_COUNT); + cur=0; + last_time=0; + while(1) { + struct seed_status *st; + apr_status_t ret = apr_queue_pop(log_queue, (void**)&st); + if(ret != APR_SUCCESS || !st) break; + if(st->status == MAPCACHE_STATUS_FINISHED) + return NULL; + if(st->status == MAPCACHE_STATUS_OK) { + failed[cur]=0; + n_metatiles_tot++; + if(!quiet) { + struct mctimeval now; + mapcache_gettimeofday(&now,NULL); + now_time = now.tv_sec + now.tv_usec / 1000000.0; + if((now_time - last_time) > 1.0) { + printf(" \r"); + printf("seeded %d tiles, now at z%d x%d y%d\r",n_metatiles_tot*tileset->metasize_x*tileset->metasize_y, st->z,st->x,st->y); + fflush(stdout); + last_time = now_time; + } + } + } else { + /* count how many errors and successes we have */ + failed[cur]=1; + nfailed=0; + ntotal=0; + if(failed_log) { + fprintf(failed_log,"%d,%d,%d\n",st->x,st->y,st->z); + } + for(i=0; i=0) ntotal++; + if(failed[i]==1) nfailed++; + } + ctx.log(&ctx, MAPCACHE_WARN, "failed to seed tile z%d,x%d,y%d:\n%s\n", st->z,st->x,st->y,st->msg); + pct = ((double)nfailed / (double)ntotal) * 100; + if(pct > percent_failed_allowed) { + ctx.log(&ctx, MAPCACHE_ERROR, "aborting seed as %.1f%% of the last %d requests failed\n", pct, FAIL_BACKLOG_COUNT); + error_detected = 1; + return NULL; + } + } + if(st->msg) free(st->msg); + free(st); + cur++; + cur %= FAIL_BACKLOG_COUNT; + } + return NULL; +} + void notice(const char *fmt, ...) { @@ -734,10 +758,12 @@ apr_getopt_t *opt; const char *configfile=NULL; apr_thread_t **threads; + apr_thread_t *log_thread; apr_threadattr_t *thread_attrs; const char *tileset_name=NULL; const char *tileset_transfer_name=NULL; const char *grid_name = NULL; + const char *cache_override = NULL; int *zooms = NULL;//[2]; mapcache_extent *extent = NULL;//[4]; int optch; @@ -752,6 +778,8 @@ double *extent_array = NULL; #ifdef USE_CLIPPERS + OGRFeatureH hFeature; + GEOSWKTReader *geoswktreader; const char *ogr_where = NULL; const char *ogr_layer = NULL; const char *ogr_sql = NULL; @@ -762,15 +790,12 @@ (void) signal(SIGINT,handle_sig_int); apr_pool_create(&ctx.pool,NULL); mapcache_context_init(&ctx); - ctx.process_pool = ctx.pool; cfg = mapcache_configuration_create(ctx.pool); ctx.config = cfg; ctx.log= mapcache_context_seeding_log; apr_getopt_init(&opt, ctx.pool, argc, argv); - seededtiles=seededtilestot=queuedtilestot=0; mapcache_gettimeofday(&starttime,NULL); - lastlogtime=starttime; argdimensions = apr_table_make(ctx.pool,3); @@ -792,6 +817,9 @@ case 'c': configfile = optarg; break; + case 'C': + cache_override = optarg; + break; case 'g': grid_name = optarg; break; @@ -810,6 +838,18 @@ return usage(argv[0],"invalid iteration mode, expecting \"drill-down\" or \"level-by-level\""); } break; + case 'L': + failed_log = fopen(optarg,"w"); + if(!failed_log) { + return usage(argv[0],"failed to open -L|--log-failed file for writing"); + } + break; + case 'R': + retry_log = fopen(optarg,"r"); + if(!retry_log) { + return usage(argv[0],"failed to open -R|--retry_failed file for writing"); + } + break; case 'm': if(!strcmp(optarg,"delete")) { mode = MAPCACHE_CMD_DELETE; @@ -835,6 +875,12 @@ #else return usage(argv[0], "multi process seeding not available on this platform"); #endif + case 'P': + percent_failed_allowed = (double)strtol(optarg, NULL, 10); + if(percent_failed_allowed<0 || percent_failed_allowed>100 ) + return usage(argv[0], "failed to parse percent, expecting number between 0 and 100"); + break; + case 'e': if ( MAPCACHE_SUCCESS != mapcache_util_extract_double_list(&ctx, (char*)optarg, ",", &extent_array, &n) || n != 4 || extent_array[0] >= extent_array[2] || extent_array[1] >= extent_array[3] ) { @@ -914,6 +960,7 @@ mapcache_configuration_post_config(&ctx,cfg); if(ctx.get_error(&ctx)) return usage(argv[0],ctx.get_error_message(&ctx)); + mapcache_connection_pool_create(&ctx.connection_pool, ctx.pool); } #ifdef USE_CLIPPERS @@ -926,6 +973,7 @@ } if(ogr_datasource) { + int f=0; OGRDataSourceH hDS = NULL; OGRLayerH layer = NULL; OGRRegisterAll(); @@ -973,21 +1021,20 @@ clippers = (const GEOSPreparedGeometry**)malloc(nClippers*sizeof(GEOSPreparedGeometry*)); - OGRFeatureH hFeature; - GEOSWKTReader *geoswktreader = GEOSWKTReader_create(); + geoswktreader = GEOSWKTReader_create(); OGR_L_ResetReading(layer); - extent = apr_palloc(ctx.pool,4*sizeof(mapcache_extent)); - int f=0; + extent = apr_palloc(ctx.pool,sizeof(mapcache_extent)); while( (hFeature = OGR_L_GetNextFeature(layer)) != NULL ) { + char *wkt; + GEOSGeometry *geosgeom; + OGREnvelope ogr_extent; OGRGeometryH geom = OGR_F_GetGeometryRef(hFeature); if(!geom || !OGR_G_IsValid(geom)) continue; - char *wkt; OGR_G_ExportToWkt(geom,&wkt); - GEOSGeometry *geosgeom = GEOSWKTReader_read(geoswktreader,wkt); + geosgeom = GEOSWKTReader_read(geoswktreader,wkt); free(wkt); clippers[f] = GEOSPrepare(geosgeom); //GEOSGeom_destroy(geosgeom); - OGREnvelope ogr_extent; OGR_G_GetEnvelope (geom, &ogr_extent); if(f == 0) { extent->minx = ogr_extent.MinX; @@ -1036,6 +1083,38 @@ return usage(argv[0],"grid not configured for tileset"); } } +#ifdef USE_CLIPPERS + if(ogr_datasource) { + /* check that the provided ogr features are compatible with the grid units */ + if(grid_link->grid->unit == MAPCACHE_UNIT_DEGREES) { + if(extent->minx < -181.0 || + extent->maxx > 181.0 || + extent->miny < -91.0 || + extent->maxy > 91.0) { + printf("\n********************************************************************************\n" + "* WARNING!!!: you are seeding a grid in latlon degreees,\n" + "* but your provided OGR intersection features span (%f,%f,%f,%f).\n" + "* this seems like an error, you should be providing OGR features that\n" + "* are in the same projection as the grid you want to seed\n" + "********************************************************************************\n\n", + extent->minx, extent->miny, extent->maxx, extent->maxy); + } + } else { + if(extent->minx > -181.0 && + extent->maxx < 181.0 && + extent->miny > -91.0 && + extent->maxy < 91.0) { + printf("\n********************************************************************************\n" + "* WARNING!!!: you are seeding a grid that is not in latlon degreees,\n" + "* but your provided OGR intersection features span (%f,%f,%f,%f) which seem to be in degrees.\n" + "* this seems like an error, you should be providing OGR features that\n" + "* are in the same projection as the grid you want to seed\n" + "********************************************************************************\n\n", + extent->minx, extent->miny, extent->maxx, extent->maxy); + } + } + } +#endif if(iteration_mode == MAPCACHE_ITERATION_UNSET) { if(!strcmp(grid_link->grid->name,"g") || !strcmp(grid_link->grid->name,"WGS84") || !strcmp(grid_link->grid->name,"GoogleMapsCompatible")) { @@ -1044,6 +1123,10 @@ iteration_mode = MAPCACHE_ITERATION_LEVEL_FIRST; } } + if(retry_log) { + iteration_mode = MAPCACHE_ITERATION_LOG; + } + if(minzoom == -1 && maxzoom == -1) { minzoom = grid_link->minz; maxzoom = grid_link->maxz - 1; @@ -1067,6 +1150,14 @@ } } + if(cache_override) { + mapcache_cache *co = mapcache_configuration_get_cache(cfg, cache_override); + if(!co) { + return usage(argv[0], "overrided cache\"%s\" to not found in configuration", cache_override); + } else { + tileset->_cache = co; + } + } } @@ -1160,11 +1251,22 @@ apr_table_set(dimensions,tileset->timedimension->key,APR_ARRAY_IDX(timedim_selected,0,char*)); } else { return usage(argv[0],"tileset references a TIME dimension, but none supplied on commandline. (hint: -D %s=",tileset->timedimension->key); - + } } } + { + /* start the logging thread */ + //create the queue where the seeding statuses will be put + apr_threadattr_t *log_thread_attrs; + apr_queue_create(&log_queue,2,ctx.pool); + + //start the rendering threads. + apr_threadattr_create(&log_thread_attrs, ctx.pool); + apr_thread_create(&log_thread, log_thread_attrs, log_thread_fn, NULL, ctx.pool); + } + if(nthreads == 0 && nprocesses == 0) { nthreads = 1; } @@ -1238,23 +1340,32 @@ apr_thread_join(&rv, threads[n]); } } - if(ctx.get_error(&ctx)) { - printf("%s",ctx.get_error_message(&ctx)); + { + struct seed_status *st = calloc(1,sizeof(struct seed_status)); + st->status = MAPCACHE_STATUS_FINISHED; + apr_queue_push(log_queue,(void*)st); + apr_thread_join(&rv, log_thread); } - if(seededtilestot>0) { + if(n_metatiles_tot>0) { struct mctimeval now_t; float duration; + int ntilestot = n_metatiles_tot*tileset->metasize_x*tileset->metasize_y; mapcache_gettimeofday(&now_t,NULL); duration = ((now_t.tv_sec-starttime.tv_sec)*1000000+(now_t.tv_usec-starttime.tv_usec))/1000000.0; - printf("\nseeded %d metatiles at %g tiles/sec\n",seededtilestot, seededtilestot/duration); + + printf("\nseeded %d metatiles (%d tiles) in %.1f seconds at %.1f tiles/sec\n",n_metatiles_tot, ntilestot, duration, ntilestot/duration); + } else { + if(!error_detected) { + printf("0 tiles needed to be seeded, exiting\n"); + } } apr_terminate(); if (error_detected > 0) { - exit(1); + exit(1); } - + return 0; } /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.2.1/Vagrantfile mapcache-1.4.1/Vagrantfile --- mapcache-1.2.1/Vagrantfile 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.4.1/Vagrantfile 2016-02-25 15:47:16.000000000 +0000 @@ -0,0 +1,27 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'socket' + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + + config.vm.hostname = "mapcache-vagrant" + + config.vm.network :forwarded_port, guest: 80, host: 8080 + + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", 1024, "--cpus", 2] + v.customize ["modifyvm", :id, "--ioapic", "on", "--largepages", "off", "--vtxvpid", "off"] + v.name = "mapcache-vagrant" + end + + config.vm.provision "shell", path: "scripts/vagrant/virtualbox-fix.sh" + config.vm.provision "shell", path: "scripts/vagrant/packages.sh" + config.vm.provision "shell", path: "scripts/vagrant/mapcache.sh" + +end