diff -Nru mapcache-1.2.1/apache/mod_mapcache.c mapcache-1.6.1/apache/mod_mapcache.c --- mapcache-1.2.1/apache/mod_mapcache.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/apache/mod_mapcache.c 2017-09-29 21:31:13.000000000 +0000 @@ -40,10 +40,6 @@ #include #include #include "mapcache.h" -#ifdef APR_HAS_THREADS -#include -apr_thread_mutex_t *thread_mutex = NULL; -#endif #ifndef _WIN32 #include @@ -59,8 +55,6 @@ typedef struct mapcache_context_apache_request mapcache_context_apache_request; typedef struct mapcache_context_apache_server mapcache_context_apache_server; -apr_pool_t *pchild = NULL; - struct mapcache_context_apache { mapcache_context ctx; }; @@ -75,6 +69,21 @@ request_rec *request; }; +typedef struct mapcache_alias_entry mapcache_alias_entry; + +struct mapcache_alias_entry { + char *endpoint; + char *configfile; + mapcache_cfg *cfg; + mapcache_connection_pool *cp; +}; + +struct mapcache_server_cfg { + apr_array_header_t *aliases; /**< list of mapcache configurations aliased to a server uri */ + apr_array_header_t *quickaliases; /**< list of mapcache configurations aliased to a server uri */ +}; +static int mapcache_alias_matches(const char *uri, const char *alias_fakename); + void apache_context_server_log(mapcache_context *c, mapcache_log_level level, char *message, ...) { mapcache_context_apache_server *ctx = (mapcache_context_apache_server*)c; @@ -168,48 +177,95 @@ return nctx; } -void init_apache_request_context(mapcache_context_apache_request *ctx) +mapcache_context_apache_request* create_apache_request_context(request_rec *r) { - mapcache_context_init((mapcache_context*)ctx); - ctx->ctx.ctx.log = apache_context_request_log; - ctx->ctx.ctx.clone = mapcache_context_request_clone; + mapcache_context_apache_request *rctx = apr_pcalloc(r->pool, sizeof(mapcache_context_apache_request)); + mapcache_context *ctx = (mapcache_context*)rctx; + mapcache_context_init(ctx); + ctx->pool = r->pool; + rctx->request = r; + ctx->log = apache_context_request_log; + ctx->clone = mapcache_context_request_clone; + return rctx; +} + +static mapcache_context_apache_server* create_apache_server_context(server_rec *s, apr_pool_t *pool) +{ + mapcache_context_apache_server *actx = apr_pcalloc(pool, sizeof(mapcache_context_apache_server)); + mapcache_context *ctx = (mapcache_context*)actx; + mapcache_context_init(ctx); + ctx->pool = pool; + ctx->config = NULL; + ctx->log = apache_context_server_log; + actx->server = s; + return actx; } -void init_apache_server_context(mapcache_context_apache_server *ctx) -{ - mapcache_context_init((mapcache_context*)ctx); - ctx->ctx.ctx.log = apache_context_server_log; -} - -static mapcache_context_apache_request* apache_request_context_create(request_rec *r) -{ - mapcache_context_apache_request *ctx = apr_pcalloc(r->pool, sizeof(mapcache_context_apache_request)); - mapcache_server_cfg *cfg = NULL; - mapcache_cfg *config = NULL; +/* read post body. code taken from "The apache modules book, Nick Kew" */ +static void read_post_body(mapcache_context_apache_request *ctx, mapcache_request_proxy *p) { + request_rec *r = ctx->request; + mapcache_context *mctx = (mapcache_context*)ctx; + int bytes,eos; + apr_bucket_brigade *bb, *bbin; + apr_bucket *b; + apr_status_t rv; + const char *clen = apr_table_get(r->headers_in, "Content-Length"); + if(clen) { + bytes = strtol(clen, NULL, 0); + if(bytes >= p->rule->max_post_len) { + mctx->set_error(mctx, HTTP_REQUEST_ENTITY_TOO_LARGE, "post request too big"); + return; + } + } else { + bytes = p->rule->max_post_len; + } - ctx->ctx.ctx.pool = r->pool; - ctx->ctx.ctx.process_pool = pchild; -#ifdef APR_HAS_THREADS - ctx->ctx.ctx.threadlock = thread_mutex; -#endif + bb = apr_brigade_create(mctx->pool, r->connection->bucket_alloc); + bbin = apr_brigade_create(mctx->pool, r->connection->bucket_alloc); + p->post_len = 0; + + do { + apr_bucket *nextb; + rv = ap_get_brigade(r->input_filters, bbin, AP_MODE_READBYTES, APR_BLOCK_READ, bytes); + if(rv != APR_SUCCESS) { + mctx->set_error(mctx, 500, "failed to read form input"); + return; + } + for(b = APR_BRIGADE_FIRST(bbin); b != APR_BRIGADE_SENTINEL(bbin); b = nextb) { + nextb = APR_BUCKET_NEXT(b); + if(APR_BUCKET_IS_EOS(b)) { + eos = 1; + } + if(!APR_BUCKET_IS_METADATA(b)) { + if(b->length != (apr_size_t)(-1)) { + p->post_len += b->length; + if(p->post_len > p->rule->max_post_len) { + apr_bucket_delete(b); + } + } + } + if(p->post_len <= p->rule->max_post_len) { + APR_BUCKET_REMOVE(b); + APR_BRIGADE_INSERT_TAIL(bb, b); + } + } + } while (!eos); - /* lookup the configuration object given the configuration file name */ - cfg = ap_get_module_config(r->server->module_config, &mapcache_module); - config = apr_hash_get(cfg->aliases,(void*)r->filename,APR_HASH_KEY_STRING); - ctx->ctx.ctx.config = config; - ctx->request = r; - init_apache_request_context(ctx); - return ctx; -} + if(p->post_len > p->rule->max_post_len) { + mctx->set_error(mctx, HTTP_REQUEST_ENTITY_TOO_LARGE, "request too big"); + return; + } -static mapcache_context_apache_server* apache_server_context_create(server_rec *s, apr_pool_t *pool) -{ - mapcache_context_apache_server *ctx = apr_pcalloc(pool, sizeof(mapcache_context_apache_server)); - ctx->ctx.ctx.pool = pool; - ctx->ctx.ctx.config = NULL; - ctx->server = s; - init_apache_server_context(ctx); - return ctx; + p->post_buf = apr_palloc(mctx->pool, p->post_len+1); + + if(p->post_len> 0) { + rv = apr_brigade_flatten(bb, p->post_buf, &(p->post_len)); + if(rv != APR_SUCCESS) { + mctx->set_error(mctx, 500, "error (flatten) reading form data"); + return; + } + } + p->post_buf[p->post_len] = 0; } static int write_http_response(mapcache_context_apache_request *ctx, mapcache_http_response *response) @@ -239,7 +295,7 @@ } } } - if(response->data) { + if(response->data && response->data->size) { ap_set_content_length(r,response->data->size); ap_rwrite((void*)response->data->buf, response->data->size, r); } @@ -251,37 +307,48 @@ static void mod_mapcache_child_init(apr_pool_t *pool, server_rec *s) { - pchild = pool; -#ifdef APR_HAS_THREADS - apr_thread_mutex_create(&thread_mutex,APR_THREAD_MUTEX_DEFAULT,pool); -#endif + for( ; s ; s=s->next) { + mapcache_server_cfg* cfg = ap_get_module_config(s->module_config, &mapcache_module); + int i,rv; + for(i=0;ialiases->nelts;i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + rv = mapcache_connection_pool_create(&(alias_entry->cp),pool); + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, "creating a child process mapcache connection pool on server %s for alias %s", s->server_hostname, alias_entry->endpoint); + if(rv!=APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "failed to create mapcache connection pool"); + } + } + for(i=0;iquickaliases->nelts;i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->quickaliases,i,mapcache_alias_entry*); + rv = mapcache_connection_pool_create(&(alias_entry->cp),pool); + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, "creating a child process mapcache connection pool on server %s for alias %s", s->server_hostname, alias_entry->endpoint); + if(rv!=APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "failed to create mapcache connection pool"); + } + } + } } -static int mod_mapcache_request_handler(request_rec *r) -{ +static int mapcache_handler(request_rec *r, mapcache_alias_entry *alias_entry) { apr_table_t *params; mapcache_request *request = NULL; - mapcache_context_apache_request *apache_ctx = NULL; + mapcache_context_apache_request *apache_ctx = create_apache_request_context(r); + mapcache_context *ctx = (mapcache_context*)apache_ctx; mapcache_http_response *http_response = NULL; - mapcache_context *global_ctx = NULL; - - if (!r->handler || strcmp(r->handler, "mapcache")) { - return DECLINED; - } - if (r->method_number != M_GET) { - return HTTP_METHOD_NOT_ALLOWED; - } + ctx->config = alias_entry->cfg; + ctx->connection_pool = alias_entry->cp; + ctx->supports_redirects = 1; + ctx->headers_in = r->headers_in; - apache_ctx = apache_request_context_create(r); - global_ctx = (mapcache_context*)apache_ctx; + params = mapcache_http_parse_param_string(ctx, r->args); - params = mapcache_http_parse_param_string(global_ctx, r->args); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "mapcache dispatch %s",r->path_info); - mapcache_service_dispatch_request(global_ctx,&request,r->path_info,params,global_ctx->config); - if(GC_HAS_ERROR(global_ctx) || !request) { + mapcache_service_dispatch_request(ctx,&request,r->path_info,params,ctx->config); + if(GC_HAS_ERROR(ctx) || !request) { return write_http_response(apache_ctx, - mapcache_core_respond_to_error(global_ctx)); + mapcache_core_respond_to_error(ctx)); } if(request->type == MAPCACHE_REQUEST_GET_CAPABILITIES) { @@ -310,31 +377,131 @@ *end = '\0'; } } - http_response = mapcache_core_get_capabilities(global_ctx,request->service,req_caps, - url,original->path_info,global_ctx->config); + http_response = mapcache_core_get_capabilities(ctx,request->service,req_caps, + url,original->path_info,ctx->config); } else if( request->type == MAPCACHE_REQUEST_GET_TILE) { mapcache_request_get_tile *req_tile = (mapcache_request_get_tile*)request; - http_response = mapcache_core_get_tile(global_ctx,req_tile); + http_response = mapcache_core_get_tile(ctx,req_tile); } else if( request->type == MAPCACHE_REQUEST_PROXY ) { + const char *buf; mapcache_request_proxy *req_proxy = (mapcache_request_proxy*)request; - http_response = mapcache_core_proxy_request(global_ctx, req_proxy); + if(r->method_number == M_POST) { + read_post_body(apache_ctx, req_proxy); + if(GC_HAS_ERROR(ctx)) { + return write_http_response(apache_ctx, mapcache_core_respond_to_error(ctx)); + } + if(!req_proxy->headers) { + req_proxy->headers = apr_table_make(ctx->pool, 2); + } + apr_table_set(req_proxy->headers, "Content-Type", r->content_type); + if((buf = apr_table_get(r->headers_in,"X-Forwarded-For"))) { +#if (AP_SERVER_MAJORVERSION_NUMBER == 2) && (AP_SERVER_MINORVERSION_NUMBER < 4) + apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(ctx->pool,"%s, %s", buf, r->connection->remote_ip)); +#else + apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(ctx->pool,"%s, %s", buf, r->connection->client_ip)); +#endif + } else { +#if (AP_SERVER_MAJORVERSION_NUMBER == 2) && (AP_SERVER_MINORVERSION_NUMBER < 4) + apr_table_set(req_proxy->headers, "X-Forwarded-For", r->connection->remote_ip); +#else + apr_table_set(req_proxy->headers, "X-Forwarded-For", r->connection->client_ip); +#endif + } + if ((buf = apr_table_get(r->headers_in, "Host"))) { + const char *buf2; + if((buf2 = apr_table_get(r->headers_in,"X-Forwarded-Host"))) { + apr_table_set(req_proxy->headers, "X-Forwarded-Host", apr_psprintf(ctx->pool,"%s, %s",buf2,buf)); + } else { + apr_table_set(req_proxy->headers, "X-Forwarded-Host", buf); + } + } + + if ((buf = apr_table_get(r->headers_in, "X-Forwarded-Server"))) { + apr_table_set(req_proxy->headers, "X-Forwarded-Server", apr_psprintf(ctx->pool, "%s, %s", buf, r->server->server_hostname)); + } else { + apr_table_set(req_proxy->headers, "X-Forwarded-Server", r->server->server_hostname); + } + } + http_response = mapcache_core_proxy_request(ctx, req_proxy); } else if( request->type == MAPCACHE_REQUEST_GET_MAP) { mapcache_request_get_map *req_map = (mapcache_request_get_map*)request; - http_response = mapcache_core_get_map(global_ctx,req_map); + http_response = mapcache_core_get_map(ctx,req_map); } else if( request->type == MAPCACHE_REQUEST_GET_FEATUREINFO) { mapcache_request_get_feature_info *req_fi = (mapcache_request_get_feature_info*)request; - http_response = mapcache_core_get_featureinfo(global_ctx,req_fi); + http_response = mapcache_core_get_featureinfo(ctx,req_fi); } else { - global_ctx->set_error(global_ctx,500,"###BUG### unknown request type"); + ctx->set_error(ctx,500,"###BUG### unknown request type"); } - if(GC_HAS_ERROR(global_ctx)) { + if(GC_HAS_ERROR(ctx)) { return write_http_response(apache_ctx, - mapcache_core_respond_to_error(global_ctx)); + mapcache_core_respond_to_error(ctx)); } return write_http_response(apache_ctx,http_response); } +static int mod_mapcache_quick_handler(request_rec *r, int lookup) { + mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias_entry *alias_entry; + int i; + + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "mapcache quick handler hook on uri %s",r->uri); + + if (!sconfig || !sconfig->quickaliases) + return DECLINED; + + if (r->uri[0] != '/' && r->uri[0]) + return DECLINED; + + if(lookup) { + return DECLINED; + } + + /* loop through the entries to find one where the alias matches */ + for(i=0; iquickaliases->nelts; i++) { + int l; + alias_entry = APR_ARRAY_IDX(sconfig->quickaliases,i,mapcache_alias_entry*); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "cheking mapcache alias %s against %s",r->uri,alias_entry->endpoint); + + if((l=mapcache_alias_matches(r->uri, alias_entry->endpoint))>0) { + if (r->method_number != M_GET && r->method_number != M_POST) { + return HTTP_METHOD_NOT_ALLOWED; + } + r->path_info = &(r->uri[l]); + return mapcache_handler(r,alias_entry); + } + } + return DECLINED; +} + +static int mod_mapcache_request_handler(request_rec *r) +{ + const char *mapcache_alias; + int i; + mapcache_server_cfg* cfg; + if (!r->handler || strcmp(r->handler, "mapcache")) { + return DECLINED; + } + if (r->method_number != M_GET && r->method_number != M_POST) { + return HTTP_METHOD_NOT_ALLOWED; + } + cfg = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias = apr_table_get(r->notes,"mapcache_alias_entry"); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "using mapcache config %s", mapcache_config_file); + if(!mapcache_alias) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "mapcache module bug? no mapcache_alias_entry found"); + return DECLINED; + } + + for(i=0; ialiases->nelts; i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + if(strcmp(alias_entry->endpoint,mapcache_alias)) + continue; + return mapcache_handler(r,alias_entry); + } + return DECLINED; /*should never happen, the fixup phase would not have oriented us here*/ +} + static int mod_mapcache_post_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, server_rec *s) { mapcache_server_cfg* cfg = ap_get_module_config(s->module_config, &mapcache_module); @@ -389,35 +556,35 @@ return urip - uri; } -static int mapcache_hook_intercept(request_rec *r) +static int mapcache_hook_fixups(request_rec *r) { - mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); - apr_hash_index_t *entry; - - if (!sconfig->aliases) - return DECLINED; - - if (r->uri[0] != '/' && r->uri[0]) - return DECLINED; - - entry = apr_hash_first(r->pool,sconfig->aliases); - - /* loop through the entries to find one where the alias matches */ - while (entry) { - int l = 0; - const char *alias; - apr_ssize_t aliaslen; - mapcache_cfg *c; - apr_hash_this(entry,(const void**)&alias,&aliaslen,(void**)&c); - - if((l=mapcache_alias_matches(r->uri, c->endpoint))>0) { - r->handler = "mapcache"; - r->filename = c->configFile; - r->path_info = &(r->uri[l]); - return OK; + int i; + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "running mapcache fixup on %s (handler:%s,filename:%s)",r->uri,r->handler,r->filename); + if(!r->handler) { + mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias_entry *alias_entry; + + if (!sconfig || !sconfig->aliases) + return DECLINED; + + if (r->uri[0] != '/' && r->uri[0]) + return DECLINED; + + /* loop through the entries to find one where the alias matches */ + for(i=0; ialiases->nelts; i++) { + int l; + alias_entry = APR_ARRAY_IDX(sconfig->aliases,i,mapcache_alias_entry*); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "cheking mapcache alias %s against %s",r->uri,alias_entry->endpoint); + + if((l=mapcache_alias_matches(r->uri, alias_entry->endpoint))>0) { + r->handler = apr_pstrdup(r->pool,"mapcache"); + apr_table_set(r->notes,"mapcache_alias_entry",alias_entry->endpoint); + r->path_info = &(r->uri[l]); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "setting config %s for alias %s",alias_entry->configfile,alias_entry->endpoint); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "using pathinfo %s from uri %s",r->path_info,r->uri); + return OK; + } } - - entry = apr_hash_next(entry); } return DECLINED; @@ -426,21 +593,19 @@ static void mod_mapcache_register_hooks(apr_pool_t *p) { - static const char * const p1[] = { "mod_alias.c", "mod_rewrite.c", NULL }; - static const char * const n1[]= { "mod_userdir.c", - "mod_vhost_alias.c", NULL - }; ap_hook_child_init(mod_mapcache_child_init, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_post_config(mod_mapcache_post_config, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_handler(mod_mapcache_request_handler, NULL, NULL, APR_HOOK_MIDDLE); - ap_hook_translate_name(mapcache_hook_intercept, p1, n1, APR_HOOK_MIDDLE); + ap_hook_quick_handler(mod_mapcache_quick_handler, NULL, NULL, APR_HOOK_MIDDLE); + ap_hook_fixups(mapcache_hook_fixups, NULL, NULL, APR_HOOK_MIDDLE); } static void* mod_mapcache_create_server_conf(apr_pool_t *pool, server_rec *s) { mapcache_server_cfg *cfg = apr_pcalloc(pool, sizeof(mapcache_server_cfg)); - cfg->aliases = NULL; + cfg->aliases = apr_array_make(pool,1,sizeof(mapcache_alias_entry*)); + cfg->quickaliases = apr_array_make(pool,1,sizeof(mapcache_alias_entry*)); return cfg; } @@ -451,44 +616,84 @@ mapcache_server_cfg *vhost = (mapcache_server_cfg*)vhost_; mapcache_server_cfg *cfg = apr_pcalloc(p,sizeof(mapcache_server_cfg)); - if (base->aliases && vhost->aliases) { - cfg->aliases = apr_hash_overlay(p, vhost->aliases,base->aliases); - } else if (vhost->aliases) { - cfg->aliases = apr_hash_copy(p,vhost->aliases); - } else if (base->aliases) { - cfg->aliases = apr_hash_copy(p,base->aliases); - } - return vhost; -} - -static const char* mapcache_add_alias(cmd_parms *cmd, void *cfg, const char *alias, const char* configfile) -{ - mapcache_server_cfg *sconfig = ap_get_module_config(cmd->server->module_config, &mapcache_module); - mapcache_cfg *config = mapcache_configuration_create(cmd->pool); - mapcache_context *ctx = (mapcache_context*)apache_server_context_create(cmd->server,cmd->pool); - char *msg = NULL; - config->configFile = apr_pstrdup(cmd->pool,configfile); - config->endpoint = alias; - mapcache_configuration_parse(ctx,configfile,config,0); + cfg->aliases = apr_array_append(p, vhost->aliases,base->aliases); + cfg->quickaliases = apr_array_append(p, vhost->quickaliases,base->quickaliases); + +#if 0 + { + mapcache_alias_entry *e; + int i; + fprintf(stderr,"#### merge called ####\n"); + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(base->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge base: have alias %s on %s\n",e->configfile,e->endpoint); + } + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(vhost->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge vhosts: have alias %s on %s\n",e->configfile,e->endpoint); + } + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge result: have alias %s on %s\n",e->configfile,e->endpoint); + } + } +#endif + return cfg; +} + +static const char* mapcache_add_alias(cmd_parms *cmd, void *cfg, const char *alias, const char* configfile, const char *quick) +{ + mapcache_server_cfg *sconfig; + mapcache_alias_entry *alias_entry; + mapcache_context *ctx; + unsigned forbidden = NOT_IN_DIRECTORY|NOT_IN_FILES; + const char *err; + +#if (AP_SERVER_MAJORVERSION_NUMBER > 2) || (AP_SERVER_MINORVERSION_NUMBER >= 4) + forbidden |= NOT_IN_HTACCESS; +#endif + + err = ap_check_cmd_context(cmd, forbidden); + if (err) { + return err; + } + + sconfig = ap_get_module_config(cmd->server->module_config, &mapcache_module); + if(!sconfig || !sconfig->aliases) + return "no mapcache module config, server bug?"; + + alias_entry = apr_pcalloc(cmd->pool,sizeof(mapcache_alias_entry)); + ctx = (mapcache_context*)create_apache_server_context(cmd->server,cmd->pool); + + alias_entry->cfg = mapcache_configuration_create(cmd->pool); + alias_entry->configfile = apr_pstrdup(cmd->pool,configfile); + alias_entry->endpoint = apr_pstrdup(cmd->pool,alias); + mapcache_configuration_parse(ctx,alias_entry->configfile,alias_entry->cfg,0); if(GC_HAS_ERROR(ctx)) { return ctx->get_error_message(ctx); } - mapcache_configuration_post_config(ctx, config); + mapcache_configuration_post_config(ctx, alias_entry->cfg); if(GC_HAS_ERROR(ctx)) { return ctx->get_error_message(ctx); } - ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on alias %s", config->configFile, alias); - if(!sconfig->aliases) { - sconfig->aliases = apr_hash_make(cmd->pool); + if(mapcache_config_services_enabled(ctx, alias_entry->cfg) <= 0) { + return "no mapcache s configured/enabled, no point in continuing."; } - apr_hash_set(sconfig->aliases,configfile,APR_HASH_KEY_STRING,config); - return msg; + if(quick && !strcmp(quick,"quick")) { + APR_ARRAY_PUSH(sconfig->quickaliases,mapcache_alias_entry*) = alias_entry; + ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on (quick) endpoint %s", alias_entry->configfile, alias_entry->endpoint); + } else { + APR_ARRAY_PUSH(sconfig->aliases,mapcache_alias_entry*) = alias_entry; + ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on endpoint %s", alias_entry->configfile, alias_entry->endpoint); + } + + return NULL; } static const command_rec mod_mapcache_cmds[] = { - AP_INIT_TAKE2("MapCacheAlias", mapcache_add_alias ,NULL,RSRC_CONF,"Aliased location of configuration file"), + AP_INIT_TAKE23("MapCacheAlias", mapcache_add_alias ,NULL,RSRC_CONF,"Aliased location of configuration file"), { NULL } } ; diff -Nru mapcache-1.2.1/cgi/mapcache.c mapcache-1.6.1/cgi/mapcache.c --- mapcache-1.2.1/cgi/mapcache.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/cgi/mapcache.c 2017-09-29 21:31:13.000000000 +0000 @@ -128,6 +128,11 @@ if(response->mtime) { char *datestr; char *if_modified_since = getenv("HTTP_IF_MODIFIED_SINCE"); + + datestr = apr_palloc(ctx->ctx.pool, APR_RFC822_DATE_LEN); + apr_rfc822_date(datestr, response->mtime); + printf("Last-Modified: %s\r\n", datestr); + if(if_modified_since) { apr_time_t ims_time; apr_int64_t ims,mtime; @@ -138,11 +143,14 @@ ims = apr_time_sec(ims_time); if(ims >= mtime) { printf("Status: 304 Not Modified\r\n"); + /* + * "The 304 response MUST NOT contain a message-body" + * https://tools.ietf.org/html/rfc2616#section-10.3.5 + */ + printf("\r\n"); + return; } } - datestr = apr_palloc(ctx->ctx.pool, APR_RFC822_DATE_LEN); - apr_rfc822_date(datestr, response->mtime); - printf("Last-Modified: %s\r\n", datestr); } if(response->data) { printf("Content-Length: %ld\r\n\r\n", response->data->size); @@ -192,12 +200,17 @@ if(GC_HAS_ERROR(ctx)) goto failed_load; mapcache_configuration_post_config(ctx, cfg); if(GC_HAS_ERROR(ctx)) goto failed_load; + if(mapcache_config_services_enabled(ctx,cfg) <= 0) { + ctx->set_error(ctx,500,"no mapcache s configured/enabled, no point in continuing."); + goto failed_load; + } /* no error, destroy the previous pool if we are reloading the config */ if(config_pool) { apr_pool_destroy(config_pool); } config_pool = tmp_config_pool; + mapcache_connection_pool_create(&ctx->connection_pool, config_pool); return; @@ -273,8 +286,6 @@ } } apr_pool_create(&(ctx->pool),config_pool); - ctx->process_pool = config_pool; - ctx->threadlock = NULL; request = NULL; pathInfo = getenv("PATH_INFO"); diff -Nru mapcache-1.2.1/cmake/FindAPACHE.cmake mapcache-1.6.1/cmake/FindAPACHE.cmake --- mapcache-1.2.1/cmake/FindAPACHE.cmake 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/cmake/FindAPACHE.cmake 2017-09-29 21:31:13.000000000 +0000 @@ -10,7 +10,7 @@ # find_path(APACHE_INCLUDE_DIR NAMES httpd.h - PATH_SUFFIXES httpd apache apache2 + PATH_SUFFIXES httpd apache apache2 apache22 apache24 ) if(NOT DEFINED APACHE_MODULE_DIR) diff -Nru mapcache-1.2.1/cmake/FindRIAK.cmake mapcache-1.6.1/cmake/FindRIAK.cmake --- mapcache-1.2.1/cmake/FindRIAK.cmake 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/cmake/FindRIAK.cmake 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,14 @@ + +FIND_PATH(RIAK_INCLUDE_DIR + NAMES riack.h +) + +FIND_LIBRARY(RIAK_LIBRARY + NAMES riack +) + +set(RIAK_INCLUDE_DIRS ${RIAK_INCLUDE_DIR}) +set(RIAK_LIBRARIES ${RIAK_LIBRARY}) +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(RIAK DEFAULT_MSG RIAK_LIBRARY RIAK_INCLUDE_DIR) +mark_as_advanced(RIAK_LIBRARY RIAK_INCLUDE_DIR) diff -Nru mapcache-1.2.1/CMakeLists.txt mapcache-1.6.1/CMakeLists.txt --- mapcache-1.2.1/CMakeLists.txt 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/CMakeLists.txt 2017-09-29 21:31:13.000000000 +0000 @@ -1,22 +1,20 @@ cmake_minimum_required (VERSION 2.6) -project (MapCache) +project (MapCache C) include(CheckFunctionExists) include(CheckIncludeFile) include(CheckCSourceCompiles) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") add_definitions(-DDEBUG) endif () set (MAPCACHE_VERSION_MAJOR 1) -set (MAPCACHE_VERSION_MINOR 2) +set (MAPCACHE_VERSION_MINOR 6) set (MAPCACHE_VERSION_REVISION 1) - if(NOT DEFINED CMAKE_INSTALL_LIBDIR) set(CMAKE_INSTALL_LIBDIR lib) endif(NOT DEFINED CMAKE_INSTALL_LIBDIR) @@ -58,19 +56,33 @@ check_function_exists("strncasecmp" HAVE_STRNCASECMP) check_function_exists("symlink" HAVE_SYMLINK) - +check_function_exists ("timegm" HAVE_TIMEGM) +check_function_exists ("strptime" HAVE_STRPTIME) set(CMAKE_SKIP_BUILD_RPATH FALSE) +if(APPLE) + set(CMAKE_MACOSX_RPATH ON) +endif() set(CMAKE_LINK_INTERFACE_LIBRARY "") file(GLOB mapcache_SOURCES lib/*.c ) +file(GLOB mapcache_HEADERS include/*.h) -add_library(mapcache SHARED ${mapcache_SOURCES}) +add_library(mapcache SHARED ${mapcache_SOURCES} ${mapcache_HEADERS}) set_target_properties(mapcache PROPERTIES VERSION ${MAPCACHE_VERSION_STRING} SOVERSION 1 ) +# Add compiler flags for warnings +if(CMAKE_COMPILER_IS_GNUCC) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=declaration-after-statement") +endif() + +if(CMAKE_C_COMPILER_ID MATCHES "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=declaration-after-statement -std=c89 -Wno-comment") +endif() + #options suported by the cmake builder option(WITH_PIXMAN "Use pixman for SSE optimized image manipulations" ON) option(WITH_SQLITE "Use sqlite as a cache backend" ON) @@ -81,11 +93,13 @@ option(WITH_GEOTIFF "Allow GeoTIFF metadata creation for TIFF cache backends" OFF) option(WITH_PCRE "Use PCRE for regex tests" OFF) option(WITH_MAPSERVER "Enable (experimental) support for the mapserver library" OFF) +option(WITH_RIAK "Use Riak as a cache backend" OFF) +option(WITH_GDAL "Choose if GDAL raster support should be built in" ON) find_package(PNG) if(PNG_FOUND) include_directories(${PNG_INCLUDE_DIR}) - target_link_libraries(mapcache ${PNG_LIBRARY}) + target_link_libraries(mapcache ${PNG_LIBRARIES}) else(PNG_FOUND) report_mandatory_not_found(PNG) endif(PNG_FOUND) @@ -150,12 +164,24 @@ endif(PIXMAN_FOUND) endif (WITH_PIXMAN) +if(WITH_GDAL) + find_package(GDAL) + if(GDAL_FOUND) + include_directories(${GDAL_INCLUDE_DIR}) + target_link_libraries(mapcache ${GDAL_LIBRARY}) + set (USE_GDAL 1) + else(GDAL_FOUND) + report_optional_not_found(GDAL) + endif(GDAL_FOUND) +endif(WITH_GDAL) + if(WITH_PCRE) find_package(PCRE) if(PCRE_FOUND) include_directories(${PCRE_INCLUDE_DIR}) target_link_libraries(mapcache ${PCRE_LIBRARY}) set (USE_PCRE 1) + add_definitions(-DPCRE_STATIC) else(PCRE_FOUND) report_optional_not_found(PCRE) endif(PCRE_FOUND) @@ -228,6 +254,16 @@ endif(MAPSERVER_FOUND) endif (WITH_MAPSERVER) +if(WITH_RIAK) + find_package(RIAK) + if(RIAK_FOUND) + include_directories(${RIAK_INCLUDE_DIR}) + target_link_libraries(mapcache ${RIAK_LIBRARY}) + set (USE_RIAK 1) + else(RIAK_FOUND) + report_optional_not_found(RIAK) + endif(RIAK_FOUND) +endif (WITH_RIAK) if(UNIX) target_link_libraries(mapcache ${CMAKE_DL_LIBS} m ) @@ -276,6 +312,8 @@ status_optional_component("Experimental TIFF write support" "${USE_TIFF_WRITE}" "${TIFF_LIBRARY}") status_optional_component("PCRE" "${USE_PCRE}" "${PCRE_LIBRARY}") status_optional_component("Experimental mapserver support" "${USE_MAPSERVER}" "${MAPSERVER_LIBRARY}") +status_optional_component("RIAK" "${USE_RIAK}" "${RIAK_LIBRARY}") +status_optional_component("GDAL" "${USE_GDAL}" "${GDAL_LIBRARY}") INSTALL(TARGETS mapcache DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff -Nru mapcache-1.2.1/debian/changelog mapcache-1.6.1/debian/changelog --- mapcache-1.2.1/debian/changelog 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/changelog 2017-11-06 02:03:11.000000000 +0000 @@ -1,3 +1,143 @@ +mapcache (1.6.1-1~trusty0) trusty; urgency=medium + + * No change rebuild for Trusty. + + -- Angelos Tzotsos Mon, 06 Nov 2017 03:00:00 +0200 + +mapcache (1.6.1-1) unstable; urgency=medium + + * New upstream release. + * Use pkg-info.mk variables instead of dpkg-parsechangelog output. + * Add autopkgtest to test installability. + * Bump Standards-Version to 4.1.1, no changes. + * Drop spelling-errors.patch, applied upstream. + * Change section for libapache2-mod-mapcache to httpd. + * Update symbols for 1.6.1. + + -- Bas Couwenberg Sat, 30 Sep 2017 09:25:05 +0200 + +mapcache (1.6.0-2) unstable; urgency=medium + + * Update symbols for other architecture. + * Move from experimental to unstable. + + -- Bas Couwenberg Sun, 18 Jun 2017 13:25:52 +0200 + +mapcache (1.6.0-1) experimental; urgency=medium + + * New upstream release. + * Fix 'contours' typo in manpage. + * Drop typo patches, applied upstream. + * Update copyright file, changes: + - Update copyright years for Regents of the University of Minnesota + - Add Frank Warmerdam to copyright holders + * Update symbols for amd64. + * Add patch to fix spelling errors. + + -- Bas Couwenberg Thu, 13 Apr 2017 18:54:23 +0200 + +mapcache (1.4.1-3) unstable; urgency=medium + + * Update symbols for hurd-i386. + * Bump Standards-Version to 3.9.8, no changes. + * Drop libmapcache2 post{inst,rm}, rely on dpkg-triggers for ldconfig. + + -- Bas Couwenberg Thu, 05 May 2016 11:40:45 +0200 + +mapcache (1.4.1-2) unstable; urgency=medium + + * Disable memcache support on hurd-i386 too. + * Update symbols for other architectures. + + -- Bas Couwenberg Fri, 26 Feb 2016 09:46:50 +0100 + +mapcache (1.4.1-1) unstable; urgency=medium + + * New upstream release. + * Update Vcs-Git URL to use HTTPS. + * Add patches for various typos. + * Bump Standards-Version to 3.9.7, no changes. + * Update symbols for 1.4.1. + + -- Bas Couwenberg Thu, 25 Feb 2016 21:04:46 +0100 + +mapcache (1.4.0-4) unstable; urgency=medium + + * Update watch file to use lowercase RC in uversionmangle. + * Rebuild for geos transition. + + -- Bas Couwenberg Sat, 29 Aug 2015 14:15:23 +0200 + +mapcache (1.4.0-3) unstable; urgency=medium + + * Disable memcache support on architectures where it's not supported. + + -- Bas Couwenberg Tue, 25 Aug 2015 19:22:22 +0200 + +mapcache (1.4.0-2) unstable; urgency=medium + + * Enable memcache support. Thanks to Frederic Junod for the patch. + (closes: 794784) + * Update Vcs-Browser URL to use HTTPS. + * Update Upstream-Contact in copyright file to include email address. + + -- Bas Couwenberg Tue, 25 Aug 2015 16:45:28 +0200 + +mapcache (1.4.0-1) unstable; urgency=medium + + * Update symbols for other architectures. + * Move from experimental to unstable. + + -- Bas Couwenberg Fri, 31 Jul 2015 00:32:07 +0200 + +mapcache (1.4.0-1~exp1) experimental; urgency=medium + + * Update watch file to support new upstream tag format too. + * New upstream release. + * Update copyright file, add license & copyright for hmac-sha.c. + * Refresh patches. + * Disable experimental mapserver support, FTBFS. + * Update symbols for amd64. + * Move docs to libapache2-mod-mapcache. + * Update mapcache_seed man page with new options. + + -- Bas Couwenberg Tue, 28 Jul 2015 21:40:01 +0200 + +mapcache (1.2.1-4) unstable; urgency=medium + + * Update symbols for other architectures. + * Move from experimental to unstable. + + -- Bas Couwenberg Sat, 25 Jul 2015 13:41:00 +0200 + +mapcache (1.2.1-4~exp1) experimental; urgency=medium + + * Add libcurl-ssl-dev as alternative for libcurl-gnutls-dev build dependency. + * Use multi-line uscan options without quotes. + * Update mapserver build dependencies for MapServer 7.0.0. + * Reorder build dependencies, add libharfbuzz-dev. + + -- Bas Couwenberg Fri, 24 Jul 2015 13:45:27 +0200 + +mapcache (1.2.1-3) unstable; urgency=medium + + * Add upstream metadata. + * Update Source URL for GitHub in copyright file. + * Update Homepage URL, strip en/ from path. + * Set the date embedded in man pages to the build date for + reproducible builds. + * Update Vcs-Browser URL to use cgit instead of gitweb. + * Update my email to @debian.org address. + * Bump Standards-Version to 3.9.6, no changes. + * Update copyright file, changes: + - Convert copyright file from iso-8859-1 to utf8 + - Update copyright years + - Drop unused files paragraph for ltmain.sh + * Drop lintian override for no-upstream-changelog, + shouldn't override pedantic tags. + + -- Bas Couwenberg Mon, 27 Apr 2015 12:11:00 +0200 + mapcache (1.2.1-2) unstable; urgency=low * Add gpb.conf to use pristine-tar by default. diff -Nru mapcache-1.2.1/debian/control mapcache-1.6.1/debian/control --- mapcache-1.2.1/debian/control 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/control 2017-09-30 07:25:05.000000000 +0000 @@ -2,44 +2,44 @@ Maintainer: Debian GIS Project Uploaders: Francesco Paolo Lovergine , Alan Boudreault , - Bas Couwenberg + Bas Couwenberg Section: devel Priority: optional Build-Depends: debhelper (>= 9), dpkg-dev (>= 1.16.1.1), - libcurl4-gnutls-dev, - libpng-dev, - libjpeg-dev, - zlib1g-dev, - libgd-dev, + dh-apache2, + apache2-dev, + chrpath, + cmake, + libcairo2-dev, + libcurl4-gnutls-dev | libcurl-ssl-dev, + libfcgi-dev, libfreetype6-dev, + libfribidi-dev, + libgd-dev, libgdal-dev (>= 1.10.0-0~), - libproj-dev, libgeos-dev (>= 3.3.1-1~), - libfribidi-dev, - libcairo2-dev, - librsvg2-dev, - apache2-dev, - dh-apache2, - libfcgi-dev, + libharfbuzz-dev, + libjpeg-dev, + libmemcached-dev, libpcre3-dev, libpixman-1-dev, + libpng-dev, + libproj-dev, + librsvg2-dev, libsqlite3-dev, libtiff-dev, - libmapserver1 (>= 6.4.0-2), - libmapserver1-dev (>= 6.4.0-2), - chrpath, - cmake, + zlib1g-dev, pkg-config, pkg-kde-tools, docbook2x, docbook-xsl, docbook-xml, xsltproc -Standards-Version: 3.9.5 -Vcs-Browser: http://anonscm.debian.org/gitweb/?p=pkg-grass/mapcache.git -Vcs-Git: git://anonscm.debian.org/pkg-grass/mapcache.git -Homepage: http://www.mapserver.org/en/mapcache/ +Standards-Version: 4.1.1 +Vcs-Browser: https://anonscm.debian.org/cgit/pkg-grass/mapcache.git +Vcs-Git: https://anonscm.debian.org/git/pkg-grass/mapcache.git +Homepage: http://www.mapserver.org/mapcache/ Package: libmapcache1 Architecture: any @@ -104,7 +104,7 @@ Package: libapache2-mod-mapcache Architecture: any -Section: web +Section: httpd Depends: ${shlibs:Depends}, ${misc:Depends} Description: tile caching server - Apache module diff -Nru mapcache-1.2.1/debian/copyright mapcache-1.6.1/debian/copyright --- mapcache-1.2.1/debian/copyright 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/copyright 2017-04-13 18:08:01.000000000 +0000 @@ -1,33 +1,31 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: MapCache -Upstream-Contact: Thomas Bonfort and the MapServer team. -Source: http://mapserver.org/ -License: MIT/X11 -Copyright: 1996-2012 Regents of the University of Minnesota. +Upstream-Contact: The MapServer team +Source: https://github.com/mapserver/mapcache/releases Files: * -Copyright: 1996-2011, Regents of the University of Minnesota - 2011, Regents of the University of Minnesota -License: MIT/X11 +Copyright: 1996-2017, Regents of the University of Minnesota + 2004, Frank Warmerdam +License: MIT Files: include/ezxml.h lib/ezxml.c Copyright: 2004-2006, Aaron Voisine License: Expat +Files: lib/hmac-sha.c +Copyright: 2005, 2007, Olivier Gay +License: BSD-3-clause + Files: lib/strptime.c Copyright: 1999, Kungliga Tekniska Högskolan License: BSD-3-clause -Files: ltmain.sh -Copyright: 1996-2001, 2003-2006 -License: GPL-2+ - Files: debian/* Copyright: 2012, Tom Payne License: BSD-3-clause -License: MIT/X11 +License: MIT Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -93,24 +91,3 @@ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -License: GPL-2+ - This program is free software; you can redistribute it - and/or modify it under the terms of the GNU General Public - License as published by the Free Software Foundation; either - version 2 of the License, or (at your option) any later - version. - . - This program is distributed in the hope that it will be - useful, but WITHOUT ANY WARRANTY; without even the implied - warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR - PURPOSE. See the GNU General Public License for more - details. - . - You should have received a copy of the GNU General Public - License along with this package; if not, write to the Free - Software Foundation, Inc., 51 Franklin St, Fifth Floor, - Boston, MA 02110-1301 USA - . - On Debian systems, the full text of the GNU General Public - License version 2 can be found in the file - `/usr/share/common-licenses/GPL-2'. diff -Nru mapcache-1.2.1/debian/docs mapcache-1.6.1/debian/docs --- mapcache-1.2.1/debian/docs 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/docs 1970-01-01 00:00:00.000000000 +0000 @@ -1 +0,0 @@ -README diff -Nru mapcache-1.2.1/debian/libapache2-mod-mapcache.docs mapcache-1.6.1/debian/libapache2-mod-mapcache.docs --- mapcache-1.2.1/debian/libapache2-mod-mapcache.docs 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/debian/libapache2-mod-mapcache.docs 2017-04-13 18:08:01.000000000 +0000 @@ -0,0 +1 @@ +README diff -Nru mapcache-1.2.1/debian/libapache2-mod-mapcache.lintian-overrides mapcache-1.6.1/debian/libapache2-mod-mapcache.lintian-overrides --- mapcache-1.2.1/debian/libapache2-mod-mapcache.lintian-overrides 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/libapache2-mod-mapcache.lintian-overrides 2017-04-13 18:08:01.000000000 +0000 @@ -1,3 +1,3 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -libapache2-mod-mapcache: no-upstream-changelog +# False positive, dependency set by dh_apache2 +libapache2-mod-mapcache: apache2-module-depends-on-real-apache2-package apache2-bin + diff -Nru mapcache-1.2.1/debian/libmapcache1-dev.lintian-overrides mapcache-1.6.1/debian/libmapcache1-dev.lintian-overrides --- mapcache-1.2.1/debian/libmapcache1-dev.lintian-overrides 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1-dev.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -libmapcache1-dev: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/libmapcache1.lintian-overrides mapcache-1.6.1/debian/libmapcache1.lintian-overrides --- mapcache-1.2.1/debian/libmapcache1.lintian-overrides 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -libmapcache1: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/libmapcache1.postinst mapcache-1.6.1/debian/libmapcache1.postinst --- mapcache-1.2.1/debian/libmapcache1.postinst 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.postinst 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -#DEBHELPER# - -if [ "$1" = configure ] ; then - ldconfig -fi - -exit 0 diff -Nru mapcache-1.2.1/debian/libmapcache1.postrm mapcache-1.6.1/debian/libmapcache1.postrm --- mapcache-1.2.1/debian/libmapcache1.postrm 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.postrm 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -#DEBHELPER# - -if [ "$1" = remove ] ; then - ldconfig -fi - -exit 0 diff -Nru mapcache-1.2.1/debian/libmapcache1.symbols mapcache-1.6.1/debian/libmapcache1.symbols --- mapcache-1.2.1/debian/libmapcache1.symbols 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.symbols 2017-09-30 07:25:05.000000000 +0000 @@ -1,4 +1,4 @@ -# SymbolsHelper-Confirmed: 1.2.0 amd64 +# SymbolsHelper-Confirmed: 1.6.1 amd64 libmapcache.so.1 #PACKAGE# #MINVER# EZXML_NIL@Base 1.0.0 _configuration_parse_wms_xml@Base 1.0.0 @@ -20,13 +20,21 @@ _create_demo_wmts@Base 1.0.0 _error_report_wmts@Base 1.0.0 _format_error_wms@Base 1.0.0 + _mapcache_cache_rest_add_headers_from_file@Base 1.6.0 + _mapcache_cache_rest_headers@Base 1.4.0 _mapcache_context_clear_error_default@Base 1.0.0 _mapcache_context_get_error_default@Base 1.0.0 _mapcache_context_get_error_msg_default@Base 1.0.0 + _mapcache_context_pop_errors@Base 1.4.0 + _mapcache_context_push_errors@Base 1.4.0 _mapcache_context_set_error_default@Base 1.0.0 _mapcache_context_set_exception_default@Base 1.0.0 _mapcache_curl_header_callback@Base 1.0.0 _mapcache_curl_memory_callback@Base 1.0.0 + _mapcache_dimension_time_get_all_entries@Base 1.6.0 + _mapcache_dimension_time_get_entries@Base 1.6.0 + _mapcache_dimension_time_get_entries_for_value@Base 1.6.0 + _mapcache_dimension_time_parse_xml@Base 1.6.0 _mapcache_imageio_classify@Base 1.0.0 _mapcache_imageio_jpeg_buffer_empty_output_buffer@Base 1.0.0 _mapcache_imageio_jpeg_buffer_term_destination@Base 1.0.0 @@ -56,21 +64,26 @@ _mapcache_source_dummy_configuration_parse_xml@Base 1.0.0 _mapcache_source_dummy_query@Base 1.0.0 _mapcache_source_dummy_render_map@Base 1.0.0 - _mapcache_source_mapserver_configuration_check@Base 1.0.0 - _mapcache_source_mapserver_configuration_parse_xml@Base 1.0.0 - _mapcache_source_mapserver_query@Base 1.0.0 - _mapcache_source_mapserver_render_map@Base 1.0.0 + _mapcache_source_fallback_configuration_check@Base 1.6.0 + _mapcache_source_fallback_configuration_parse_xml@Base 1.6.0 + _mapcache_source_fallback_query@Base 1.6.0 + _mapcache_source_fallback_render_map@Base 1.6.0 + _mapcache_source_gdal_configuration_check@Base 1.6.0 + _mapcache_source_gdal_configuration_parse@Base 1.6.0 + _mapcache_source_gdal_render_metatile@Base 1.6.0 _mapcache_source_wms_configuration_check@Base 1.0.0 _mapcache_source_wms_configuration_parse_xml@Base 1.0.0 _mapcache_source_wms_query@Base 1.0.0 _mapcache_source_wms_render_map@Base 1.0.0 - _mapcache_timedimension_sqlite_get_all_entries@Base 1.2.0 - _mapcache_timedimension_sqlite_get_entries@Base 1.2.0 - _mapcache_timedimension_sqlite_parse_xml@Base 1.2.0 _mapcache_unescape_url@Base 1.0.0 _wmts_inspire_metadata_responselanguages@Base 1.2.0 _wmts_service_identification_keywords@Base 1.2.0 + base64_encode@Base 1.4.0 + buffer_write_callback@Base 1.4.0 crc_table@Base 1.2.0 + (arch=!alpha !amd64 !arm64 !armel !armhf !hurd-i386 !i386 !kfreebsd-amd64 !kfreebsd-i386 !mips64el !mipsel !ppc64el !x32)crc_table_computed@Base 1.4.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)create_memcache@Base 1.4.1 + empty_png_384@Base 1.4.0 ezxml_add_child@Base 1.0.0 ezxml_ampencode@Base 1.0.0 ezxml_attr@Base 1.0.0 @@ -104,14 +117,41 @@ ezxml_toxml_r@Base 1.0.0 ezxml_vget@Base 1.0.0 fontData@Base 1.0.0 + hmac_sha1@Base 1.4.0 + hmac_sha256@Base 1.4.0 + hmac_sha256_final@Base 1.4.0 + hmac_sha256_init@Base 1.4.0 + hmac_sha256_reinit@Base 1.4.0 + hmac_sha256_update@Base 1.4.0 lock_filename_for_resource@Base 1.0.0 mapcache_assemble_maps@Base 1.2.0 mapcache_buffer_append@Base 1.0.0 mapcache_buffer_create@Base 1.0.0 + mapcache_cache_azure_create@Base 1.4.0 + mapcache_cache_bdb_create@Base 1.6.0 + mapcache_cache_composite_create@Base 1.4.0 + mapcache_cache_couchbase_create@Base 1.6.0 mapcache_cache_disk_create@Base 1.0.0 + mapcache_cache_fallback_create@Base 1.4.0 + mapcache_cache_google_create@Base 1.4.0 mapcache_cache_mbtiles_create@Base 1.0.0 + mapcache_cache_memcache_create@Base 1.6.0 + mapcache_cache_multitier_create@Base 1.4.0 + mapcache_cache_rest_create@Base 1.4.0 + mapcache_cache_rest_init@Base 1.4.0 + mapcache_cache_riak_create@Base 1.6.0 + mapcache_cache_s3_create@Base 1.4.0 mapcache_cache_sqlite_create@Base 1.0.0 + mapcache_cache_tc_create@Base 1.6.0 mapcache_cache_tiff_create@Base 1.0.0 + mapcache_cache_tile_delete@Base 1.6.0 + mapcache_cache_tile_exists@Base 1.6.0 + mapcache_cache_tile_get@Base 1.6.0 + mapcache_cache_tile_multi_set@Base 1.6.0 + mapcache_cache_tile_set@Base 1.6.0 + mapcache_config_parse_locker@Base 1.4.0 + mapcache_config_parse_locker_old@Base 1.6.0 + mapcache_config_services_enabled@Base 1.6.0 mapcache_configuration_add_cache@Base 1.0.0 mapcache_configuration_add_grid@Base 1.0.0 mapcache_configuration_add_image_format@Base 1.0.0 @@ -126,6 +166,10 @@ mapcache_configuration_parse@Base 1.0.0 mapcache_configuration_parse_xml@Base 1.0.0 mapcache_configuration_post_config@Base 1.0.0 + mapcache_connection_pool_create@Base 1.4.0 + mapcache_connection_pool_get_connection@Base 1.4.0 + mapcache_connection_pool_invalidate_connection@Base 1.4.0 + mapcache_connection_pool_release_connection@Base 1.4.0 mapcache_context_copy@Base 1.0.0 mapcache_context_init@Base 1.0.0 mapcache_core_get_capabilities@Base 1.0.0 @@ -134,23 +178,23 @@ mapcache_core_get_tile@Base 1.0.0 mapcache_core_proxy_request@Base 1.0.0 mapcache_core_respond_to_error@Base 1.0.0 - mapcache_dimension_intervals_create@Base 1.0.0 mapcache_dimension_regex_create@Base 1.0.0 + mapcache_dimension_sqlite_create@Base 1.4.0 mapcache_dimension_time_create@Base 1.0.0 mapcache_dimension_values_create@Base 1.0.0 mapcache_empty_png_decode@Base 1.2.0 mapcache_error_image@Base 1.0.0 -#MISSING: 1.2.0# mapcache_fetch_maps@Base 1.0.0 mapcache_grid_compute_limits@Base 1.0.0 mapcache_grid_create@Base 1.0.0 mapcache_grid_get_cell@Base 1.0.0 - mapcache_grid_get_closest_level@Base 1.0.0 + mapcache_grid_get_closest_wms_level@Base 1.4.0 mapcache_grid_get_crs@Base 1.0.0 - mapcache_grid_get_extent@Base 1.0.0 mapcache_grid_get_horizontal_resolution@Base 1.0.0 mapcache_grid_get_level@Base 1.0.0 + mapcache_grid_get_metatile_extent@Base 1.6.0 mapcache_grid_get_resolution@Base 1.0.0 mapcache_grid_get_srs@Base 1.0.0 + mapcache_grid_get_tile_extent@Base 1.6.0 mapcache_grid_get_vertical_resolution@Base 1.0.0 mapcache_grid_get_xy@Base 1.0.0 mapcache_http_build_url@Base 1.0.0 @@ -166,6 +210,7 @@ mapcache_image_create@Base 1.0.0 mapcache_image_create_empty@Base 1.0.0 mapcache_image_create_with_data@Base 1.2.0 + mapcache_image_fill@Base 1.4.0 mapcache_image_has_alpha@Base 1.0.0 mapcache_image_merge@Base 1.0.0 mapcache_image_metatile_split@Base 1.0.0 @@ -179,9 +224,32 @@ mapcache_imageio_is_valid_format@Base 1.0.0 mapcache_is_axis_inverted@Base 1.0.0 mapcache_lock_or_wait_for_resource@Base 1.0.0 + mapcache_locker_disk_aquire_lock@Base 1.4.0 + mapcache_locker_disk_create@Base 1.4.0 + mapcache_locker_disk_parse_xml@Base 1.4.0 + mapcache_locker_disk_ping_lock@Base 1.4.0 + mapcache_locker_disk_release_lock@Base 1.4.0 + mapcache_locker_fallback_aquire_lock@Base 1.4.0 + mapcache_locker_fallback_create@Base 1.4.0 + mapcache_locker_fallback_parse_xml@Base 1.4.0 + mapcache_locker_fallback_ping_lock@Base 1.4.0 + mapcache_locker_fallback_release_lock@Base 1.4.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_aquire_lock@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_create@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_parse_xml@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_ping_lock@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_release_lock@Base 1.4.1 + mapcache_make_parent_dirs@Base 1.4.1 + mapcache_map_set_cached_dimension@Base 1.6.0 + mapcache_map_set_requested_dimension@Base 1.6.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_memcache_connection_constructor@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_memcache_connection_destructor@Base 1.4.1 mapcache_meters_per_unit@Base 1.0.0 mapcache_ogc_strptime@Base 1.2.0 mapcache_prefetch_tiles@Base 1.0.0 + mapcache_requested_dimensions_clone@Base 1.6.0 + mapcache_rest_connection_constructor@Base 1.6.0 + mapcache_rest_connection_destructor@Base 1.6.0 mapcache_service_demo_create@Base 1.0.0 mapcache_service_dispatch_request@Base 1.0.0 mapcache_service_gmaps_create@Base 1.0.0 @@ -191,11 +259,24 @@ mapcache_service_ve_create@Base 1.0.0 mapcache_service_wms_create@Base 1.0.0 mapcache_service_wmts_create@Base 1.0.0 + mapcache_set_cached_dimension@Base 1.6.0 + mapcache_set_requested_dimension@Base 1.6.0 mapcache_source_dummy_create@Base 1.0.0 + mapcache_source_fallback_create@Base 1.6.0 + mapcache_source_gdal_connection_constructor@Base 1.6.0 + mapcache_source_gdal_connection_destructor@Base 1.6.0 mapcache_source_gdal_create@Base 1.0.0 mapcache_source_init@Base 1.0.0 mapcache_source_mapserver_create@Base 1.0.0 + mapcache_source_query_info@Base 1.6.0 + mapcache_source_render_map@Base 1.6.0 mapcache_source_wms_create@Base 1.0.0 + mapcache_sqlite_connection_constructor@Base 1.4.0 + mapcache_sqlite_connection_destructor@Base 1.4.0 + mapcache_sqlite_dimension_connection_constructor@Base 1.4.0 + mapcache_sqlite_dimension_connection_destructor@Base 1.4.0 + mapcache_tile_set_cached_dimension@Base 1.6.0 + mapcache_tile_set_requested_dimension@Base 1.6.0 mapcache_tileset_add_watermark@Base 1.0.0 mapcache_tileset_assemble_map_tiles@Base 1.0.0 mapcache_tileset_assemble_out_of_zoom_tile@Base 1.2.0 @@ -214,16 +295,24 @@ mapcache_tileset_tile_create@Base 1.0.0 mapcache_tileset_tile_delete@Base 1.0.0 mapcache_tileset_tile_get@Base 1.0.0 + mapcache_tileset_tile_get_readonly@Base 1.6.0 + mapcache_tileset_tile_get_with_subdimensions@Base 1.6.0 + mapcache_tileset_tile_resource_key@Base 1.6.0 + mapcache_tileset_tile_set_get_with_subdimensions@Base 1.6.0 mapcache_tileset_tile_validate@Base 1.0.0 - mapcache_timedimension_get_entries_for_value@Base 1.2.0 - mapcache_timedimension_sqlite_create@Base 1.2.0 + mapcache_tileset_tile_validate_x@Base 1.6.0 + mapcache_tileset_tile_validate_y@Base 1.6.0 + mapcache_tileset_tile_validate_z@Base 1.6.0 mapcache_unlock_resource@Base 1.0.0 mapcache_util_extract_double_list@Base 1.0.0 mapcache_util_extract_int_list@Base 1.0.0 mapcache_util_get_tile_dimkey@Base 1.0.0 mapcache_util_get_tile_key@Base 1.0.0 + mapcache_util_quadkey_decode@Base 1.6.0 + mapcache_util_quadkey_encode@Base 1.6.0 mapcache_util_str_replace@Base 1.0.0 mapcache_util_str_sanitize@Base 1.0.0 + mapcache_util_str_xml_escape@Base 1.6.1 parseCache@Base 1.0.0 parseDimensions@Base 1.0.0 parseFormat@Base 1.0.0 @@ -232,5 +321,14 @@ parseServices@Base 1.0.0 parseSource@Base 1.0.0 parseTileset@Base 1.0.0 - parseTimeDimension@Base 1.2.0 relative_path@Base 1.0.0 + sha256@Base 1.4.0 + sha256_final@Base 1.4.0 + sha256_h0@Base 1.4.0 + sha256_init@Base 1.4.0 + sha256_k@Base 1.4.0 + sha256_transf@Base 1.4.0 + sha256_update@Base 1.4.0 + sha_hex_encode@Base 1.4.0 + to_hex@Base 1.4.0 + url_encode@Base 1.4.0 diff -Nru mapcache-1.2.1/debian/mapcache-cgi.lintian-overrides mapcache-1.6.1/debian/mapcache-cgi.lintian-overrides --- mapcache-1.2.1/debian/mapcache-cgi.lintian-overrides 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/mapcache-cgi.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -mapcache-cgi: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/mapcache_seed.1.xml mapcache-1.6.1/debian/mapcache_seed.1.xml --- mapcache-1.2.1/debian/mapcache_seed.1.xml 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/mapcache_seed.1.xml 2017-04-13 18:08:01.000000000 +0000 @@ -45,6 +45,14 @@ + + override + + Override cache used by selected tileset (useful for selectively seeding fallback/multitier caches). + + + + name @@ -125,6 +133,30 @@ + + number + + Percent of failed requests allowed from the last 1000 before we abort (default: 1%, set to 0 to abort on first error). + + + + + + file + + Log failed tiles to file. + + + + + + file + + Rtry failed requests logged to file by . + + + + seed|delete|transfer @@ -291,7 +323,7 @@ - Given a shapefile that contains the world country countours, seed only + Given a shapefile that contains the world country contours, seed only the areas that are covered by land (i.e. skip the oceans). Also use 4 request threads in parallel: diff -Nru mapcache-1.2.1/debian/mapcache-tools.lintian-overrides mapcache-1.6.1/debian/mapcache-tools.lintian-overrides --- mapcache-1.2.1/debian/mapcache-tools.lintian-overrides 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/mapcache-tools.lintian-overrides 1970-01-01 00:00:00.000000000 +0000 @@ -1,3 +0,0 @@ -# No changes in source, you can compare release tags online: -# https://github.com/mapserver/mapcache/compare/rel-1-2-0...rel-1-2-1 -mapcache-tools: no-upstream-changelog diff -Nru mapcache-1.2.1/debian/patches/cmake-mapserver-include.patch mapcache-1.6.1/debian/patches/cmake-mapserver-include.patch --- mapcache-1.2.1/debian/patches/cmake-mapserver-include.patch 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/patches/cmake-mapserver-include.patch 2017-04-13 18:08:01.000000000 +0000 @@ -1,9 +1,10 @@ Description: Include mapserver cmake package for additional include directories. -Author: Bas Couwenberg -Last-Update: 2013-10-21 +Author: Bas Couwenberg +Forwarded: not-needed + --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -218,10 +218,18 @@ if(WITH_GEOTIFF) +@@ -244,10 +244,18 @@ if(WITH_GEOTIFF) endif (WITH_GEOTIFF) if(WITH_MAPSERVER) diff -Nru mapcache-1.2.1/debian/rules mapcache-1.6.1/debian/rules --- mapcache-1.2.1/debian/rules 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/rules 2017-09-27 13:14:44.000000000 +0000 @@ -10,7 +10,8 @@ # Enable hardening build flags export DEB_BUILD_MAINT_OPTIONS=hardening=+all -DEB_HOST_MULTIARCH?=$(shell dpkg-architecture -qDEB_HOST_MULTIARCH) +DEB_HOST_ARCH ?= $(shell dpkg-architecture -qDEB_HOST_ARCH) +DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH) CPPFLAGS:=$(shell dpkg-buildflags --get CPPFLAGS) CFLAGS:=$(shell dpkg-buildflags --get CFLAGS) @@ -19,7 +20,11 @@ CFLAGS+=$(CPPFLAGS) CFLAGS+=$(LDFLAGS) -MAPCACHE_VERSION = $(shell dpkg-parsechangelog | sed -ne 's/^Version: \(.*\)-.*/\1/p' | sed -e 's/^\(.*\)~.*/\1/' | sed -e 's/dev.*/.0/') +include /usr/share/dpkg/pkg-info.mk + +UPSTREAM_VERSION = $(shell echo $(DEB_VERSION_UPSTREAM) | sed -e 's/\+.*//') + +BUILD_DATE=$(shell LC_ALL=C date -u "+%d %B %Y" -d "@$(SOURCE_DATE_EPOCH)") CMAKE_OPTS:= \ -DCMAKE_INSTALL_PREFIX=/usr \ @@ -27,11 +32,10 @@ -DWITH_PIXMAN=1 \ -DWITH_SQLITE=1 \ -DWITH_BERKELEY_DB=0 \ - -DWITH_MEMCACHE=0 \ -DWITH_TIFF=1 \ -DWITH_TIFF_WRITE_SUPPORT=0 \ -DWITH_GEOTIFF=0 \ - -DWITH_MAPSERVER=1 \ + -DWITH_MAPSERVER=0 \ -DWITH_PCRE=1 \ -DWITH_APACHE=1 \ -DWITH_VERSION_STRING=1 \ @@ -40,6 +44,13 @@ -DWITH_GEOS=1 \ -DWITH_OGR=1 +# Disable memcache support on architectures where it's not supported +ifneq (,$(findstring $(DEB_HOST_ARCH),"armel armhf i386 mips mipsel powerpc hppa hurd-i386")) + CMAKE_OPTS += -DWITH_MEMCACHE=0 +else + CMAKE_OPTS += -DWITH_MEMCACHE=1 +endif + %: dh $@ --with apache2,pkgkde_symbolshelper \ --parallel \ @@ -57,7 +68,7 @@ override_dh_auto_build: # Create man page from DocBook XML - -docbook2x-man $(CURDIR)/debian/mapcache_seed.1.xml + -docbook2x-man --string-param header-3="$(BUILD_DATE)" $(CURDIR)/debian/mapcache_seed.1.xml -mv mapcache_seed.1 $(CURDIR)/debian/ dh_auto_build @@ -89,5 +100,5 @@ dh_install --autodest --list-missing override_dh_makeshlibs: - dh_makeshlibs -- -c0 -v$(MAPCACHE_VERSION) + dh_makeshlibs -- -c0 -v$(UPSTREAM_VERSION) diff -Nru mapcache-1.2.1/debian/tests/control mapcache-1.6.1/debian/tests/control --- mapcache-1.2.1/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/debian/tests/control 2017-09-27 13:15:10.000000000 +0000 @@ -0,0 +1,3 @@ +# Test installability +Depends: @ +Test-Command: /bin/true diff -Nru mapcache-1.2.1/debian/upstream/metadata mapcache-1.6.1/debian/upstream/metadata --- mapcache-1.2.1/debian/upstream/metadata 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/debian/upstream/metadata 2017-04-13 18:08:01.000000000 +0000 @@ -0,0 +1,7 @@ +--- +Bug-Database: https://github.com/mapserver/mapcache/issues +Bug-Submit: https://github.com/mapserver/mapcache/issues/new +Contact: The MapServer team +Name: MapCache +Repository: https://github.com/mapserver/mapcache.git +Repository-Browse: https://github.com/mapserver/mapcache diff -Nru mapcache-1.2.1/debian/watch mapcache-1.6.1/debian/watch --- mapcache-1.2.1/debian/watch 2014-02-28 23:07:58.000000000 +0000 +++ mapcache-1.6.1/debian/watch 2017-04-13 18:08:01.000000000 +0000 @@ -1,3 +1,7 @@ version=3 -opts="dversionmangle=s/\+(debian|dfsg|ds|deb)\d*$//,uversionmangle=s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha)\d*)$/$1~$2/; s/\-/\./g" \ -https://github.com/mapserver/mapcache/releases (?:.*/)?(?:rel|v|mapcache)[\-\_](\d[\d\-\.]+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) +opts=\ +dversionmangle=s/\+(debian|dfsg|ds|deb)\d*$//,\ +uversionmangle=s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha)\d*)$/$1~$2/;s/RC/rc/;s/\-/\./g,\ +filenamemangle=s/(?:.*?)?(?:rel|v|mapcache)?[\-\_]?(\d\S+)\.(tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz)))/mapcache-$1.$2/ \ +https://github.com/mapserver/mapcache/releases \ +(?:.*?/)?(?:rel|v|mapcache)?[\-\_]?(\d[\d\-\.]+)\.(?:tgz|tbz|txz|(?:tar\.(?:gz|bz2|xz))) diff -Nru mapcache-1.2.1/.gitignore mapcache-1.6.1/.gitignore --- mapcache-1.2.1/.gitignore 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/.gitignore 2017-09-29 21:31:13.000000000 +0000 @@ -1,3 +1,5 @@ .*.swp nbproject/ /build/ +/build_vagrant/ +/.vagrant/ diff -Nru mapcache-1.2.1/include/mapcache-config.h.in mapcache-1.6.1/include/mapcache-config.h.in --- mapcache-1.2.1/include/mapcache-config.h.in 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/include/mapcache-config.h.in 2017-09-29 21:31:13.000000000 +0000 @@ -11,9 +11,12 @@ #cmakedefine USE_GEOTIFF 1 #cmakedefine USE_PCRE 1 #cmakedefine USE_MAPSERVER 1 +#cmakedefine USE_RIAK 1 +#cmakedefine USE_GDAL 1 #cmakedefine HAVE_STRNCASECMP 1 #cmakedefine HAVE_SYMLINK 1 - +#cmakedefine HAVE_STRPTIME 1 +#cmakedefine HAVE_TIMEGM 1 #endif diff -Nru mapcache-1.2.1/include/mapcache.h mapcache-1.6.1/include/mapcache.h --- mapcache-1.2.1/include/mapcache.h 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/include/mapcache.h 2017-09-29 21:31:13.000000000 +0000 @@ -1,7 +1,6 @@ /****************************************************************************** - * $Id$ * - * Project: MapServer + * Project: MapCache * Author: Thomas Bonfort and the MapServer team. * ****************************************************************************** @@ -39,32 +38,18 @@ #include #include +#include #include "util.h" #include "ezxml.h" #include "errors.h" -#if 0 -#ifdef USE_GDAL -#include -#include -#endif -#endif #include #include #include -#ifdef USE_PCRE -#include -#else -#include -#endif - -#ifdef USE_MEMCACHE -#include -#endif #define MAPCACHE_SUCCESS 0 #define MAPCACHE_FAILURE 1 @@ -75,11 +60,17 @@ #define MAPCACHE_TILESET_WRONG_EXTENT 4 #define MAPCACHE_CACHE_MISS 5 #define MAPCACHE_FILE_LOCKED 6 +#define MAPCACHE_CACHE_RELOAD 7 -#define MAPCACHE_USERAGENT "mod-mapcache/"MAPCACHE_VERSION +#define MAPCACHE_MAX_NUM_TILES 1000 -#define MAPCACHE_LOCKFILE_PREFIX "_gc_lock" +#define MAPCACHE_USERAGENT "mod-mapcache/"MAPCACHE_VERSION +#if defined(_WIN32) && !defined(__CYGWIN__) +# define MS_DLL_EXPORT __declspec(dllexport) +#else +#define MS_DLL_EXPORT +#endif typedef struct mapcache_image_format mapcache_image_format; @@ -98,36 +89,16 @@ typedef struct mapcache_request_get_feature_info mapcache_request_get_feature_info; typedef struct mapcache_map mapcache_map; typedef struct mapcache_http_response mapcache_http_response; -typedef struct mapcache_source_wms mapcache_source_wms; -#if 0 -typedef struct mapcache_source_gdal mapcache_source_gdal; -#endif -typedef struct mapcache_cache_disk mapcache_cache_disk; -#ifdef USE_TIFF -typedef struct mapcache_cache_tiff mapcache_cache_tiff; -#endif typedef struct mapcache_http mapcache_http; typedef struct mapcache_request mapcache_request; +typedef struct mapcache_request_image mapcache_request_image; typedef struct mapcache_request_proxy mapcache_request_proxy; typedef struct mapcache_request_get_capabilities mapcache_request_get_capabilities; -typedef struct mapcache_request_get_capabilities_demo mapcache_request_get_capabilities_demo; -typedef struct mapcache_request_get_capabilities_wms mapcache_request_get_capabilities_wms; -typedef struct mapcache_request_get_capabilities_wmts mapcache_request_get_capabilities_wmts; typedef struct mapcache_forwarding_rule mapcache_forwarding_rule; -typedef struct mapcache_request_get_capabilities_tms mapcache_request_get_capabilities_tms; -typedef struct mapcache_request_get_capabilities_kml mapcache_request_get_capabilities_kml; typedef struct mapcache_request_get_tile mapcache_request_get_tile; typedef struct mapcache_request_get_map mapcache_request_get_map; typedef struct mapcache_service mapcache_service; -typedef struct mapcache_service_wms mapcache_service_wms; -typedef struct mapcache_service_wmts mapcache_service_wmts; -typedef struct mapcache_service_gmaps mapcache_service_gmaps; -typedef struct mapcache_service_ve mapcache_service_ve; -typedef struct mapcache_service_tms mapcache_service_tms; -typedef struct mapcache_service_kml mapcache_service_kml; -typedef struct mapcache_service_mapguide mapcache_service_mapguide; -typedef struct mapcache_service_demo mapcache_service_demo; typedef struct mapcache_server_cfg mapcache_server_cfg; typedef struct mapcache_image mapcache_image; typedef struct mapcache_grid mapcache_grid; @@ -135,13 +106,27 @@ typedef struct mapcache_grid_link mapcache_grid_link; typedef struct mapcache_context mapcache_context; typedef struct mapcache_dimension mapcache_dimension; -typedef struct mapcache_dimension_time mapcache_dimension_time; -typedef struct mapcache_timedimension mapcache_timedimension; -typedef struct mapcache_dimension_intervals mapcache_dimension_intervals; -typedef struct mapcache_dimension_values mapcache_dimension_values; -typedef struct mapcache_dimension_regex mapcache_dimension_regex; +typedef struct mapcache_requested_dimension mapcache_requested_dimension; typedef struct mapcache_extent mapcache_extent; typedef struct mapcache_extent_i mapcache_extent_i; +typedef struct mapcache_connection_pool mapcache_connection_pool; +typedef struct mapcache_locker mapcache_locker; + +typedef enum { + MAPCACHE_DIMENSION_VALUES, + MAPCACHE_DIMENSION_REGEX, + MAPCACHE_DIMENSION_INTERVALS, + MAPCACHE_DIMENSION_TIME, + MAPCACHE_DIMENSION_SQLITE +} mapcache_dimension_type; + +typedef enum { + MAPCACHE_DIMENSION_ASSEMBLY_NONE, + MAPCACHE_DIMENSION_ASSEMBLY_STACK, + MAPCACHE_DIMENSION_ASSEMBLY_ANIMATE +} mapcache_dimension_assembly_type; + + /** \defgroup utility Utility */ /** @{ */ @@ -181,7 +166,7 @@ void (*set_exception)(mapcache_context *ctx, char *key, char *message, ...); /** - * \brief query context to know if an error has occured + * \brief query context to know if an error has occurred * \memberof mapcache_context */ int (*get_error)(mapcache_context * ctx); @@ -198,6 +183,18 @@ */ void (*clear_errors)(mapcache_context * ctx); + /** + * \brief clear current error and store it in mapcache_error + * \memberof mapcache_context + */ + void (*pop_errors)(mapcache_context * ctx, void **error); + + /** + * \brief restore error status from mapcache_error + * \memberof mapcache_context + */ + void (*push_errors)(mapcache_context * ctx, void *error); + /** * \brief log a message @@ -208,20 +205,21 @@ const char* (*get_instance_id)(mapcache_context * ctx); mapcache_context* (*clone)(mapcache_context *ctx); apr_pool_t *pool; - apr_pool_t *process_pool; - void *threadlock; + mapcache_connection_pool *connection_pool; char *_contenttype; char *_errmsg; int _errcode; mapcache_cfg *config; mapcache_service *service; apr_table_t *exceptions; + int supports_redirects; + apr_table_t *headers_in; }; -void mapcache_context_init(mapcache_context *ctx); -void mapcache_context_copy(mapcache_context *src, mapcache_context *dst); +MS_DLL_EXPORT void mapcache_context_init(mapcache_context *ctx); +MS_DLL_EXPORT void mapcache_context_copy(mapcache_context *src, mapcache_context *dst); -#define GC_CHECK_ERROR_RETURN(ctx) (if(((mapcache_context*)ctx)->_errcode) return MAPCACHE_FAILURE;) +#define GC_CHECK_ERROR_RETURN(ctx) if(((mapcache_context*)ctx)->_errcode) return MAPCACHE_FAILURE; #define GC_CHECK_ERROR(ctx) if(((mapcache_context*)ctx)->_errcode) return; #define GC_HAS_ERROR(ctx) (((mapcache_context*)ctx)->_errcode > 0) @@ -267,7 +265,8 @@ MAPCACHE_SOURCE_WMS, MAPCACHE_SOURCE_MAPSERVER, MAPCACHE_SOURCE_DUMMY, - MAPCACHE_SOURCE_GDAL + MAPCACHE_SOURCE_GDAL, + MAPCACHE_SOURCE_FALLBACK } mapcache_source_type; /**\interface mapcache_source @@ -278,6 +277,8 @@ mapcache_extent data_extent; /**< extent in which this source can produce data */ mapcache_source_type type; apr_table_t *metadata; + unsigned int retry_count; + double retry_delay; apr_array_header_t *info_formats; /** @@ -285,11 +286,11 @@ * * sets the mapcache_metatile::tile::data for the given tile */ - void (*render_map)(mapcache_context *ctx, mapcache_map *map); + void (*_render_map)(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map); - void (*query_info)(mapcache_context *ctx, mapcache_feature_info *fi); + void (*_query_info)(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi); - void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_source * source); + void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_source * source, mapcache_cfg *config); void (*configuration_check)(mapcache_context *ctx, mapcache_cfg *cfg, mapcache_source * source); }; @@ -299,56 +300,14 @@ struct mapcache_http { char *url; /**< the base url to request */ apr_table_t *headers; /**< additional headers to add to the http request, eg, Referer */ + char *post_body; + size_t post_len; int connection_timeout; int timeout; /* TODO: authentication */ }; -/**\class mapcache_source_wms - * \brief WMS mapcache_source - * \implements mapcache_source - */ -struct mapcache_source_wms { - mapcache_source source; - apr_table_t *wms_default_params; /**< default WMS parameters (SERVICE,REQUEST,STYLES,VERSION) */ - apr_table_t *getmap_params; /**< WMS parameters specified in configuration */ - apr_table_t *getfeatureinfo_params; /**< WMS parameters specified in configuration */ - mapcache_http *http; -}; -#ifdef USE_MAPSERVER -/**\class mapcache_source_mapserver - * \brief WMS mapcache_source - * \implements mapcache_source - */ -typedef struct mapcache_source_mapserver mapcache_source_mapserver; -struct mapcache_source_mapserver { - mapcache_source source; - char *mapfile; -}; -#endif - -typedef struct mapcache_source_dummy mapcache_source_dummy; -struct mapcache_source_dummy { - mapcache_source source; - char *mapfile; - void *mapobj; -}; -#if 0 -#ifdef USE_GDAL -/**\class mapcache_source_gdal - * \brief GDAL mapcache_source - * \implements mapcache_source - */ -struct mapcache_source_gdal { - mapcache_source source; - char *datastr; /**< the gdal source string*/ - apr_table_t *gdal_params; /**< GDAL parameters specified in configuration */ - GDALDatasetH *poDataset; -}; -#endif -/** @} */ -#endif /** \defgroup cache Caches */ @@ -356,21 +315,15 @@ /** @{ */ typedef enum { MAPCACHE_CACHE_DISK -#ifdef USE_MEMCACHE + ,MAPCACHE_CACHE_REST ,MAPCACHE_CACHE_MEMCACHE -#endif -#ifdef USE_SQLITE ,MAPCACHE_CACHE_SQLITE -#endif -#ifdef USE_BDB ,MAPCACHE_CACHE_BDB -#endif -#ifdef USE_TC ,MAPCACHE_CACHE_TC -#endif -#ifdef USE_TIFF ,MAPCACHE_CACHE_TIFF -#endif + ,MAPCACHE_CACHE_COMPOSITE + ,MAPCACHE_CACHE_COUCHBASE + ,MAPCACHE_CACHE_RIAK } mapcache_cache_type; /** \interface mapcache_cache @@ -380,135 +333,77 @@ char *name; /**< key this cache is referenced by */ mapcache_cache_type type; apr_table_t *metadata; + unsigned int retry_count; + double retry_delay; + /** * get tile content from cache - * \returns MAPCACHE_SUCCESS if the data was correctly loaded from the disk + * \returns MAPCACHE_SUCCESS if the data was correctly loaded from the cache * \returns MAPCACHE_FAILURE if the file exists but contains no data - * \returns MAPCACHE_CACHE_MISS if the file does not exist on the disk + * \returns MAPCACHE_CACHE_MISS if the file does not exist on the cache * \memberof mapcache_cache */ - int (*tile_get)(mapcache_context *ctx, mapcache_tile * tile); + int (*_tile_get)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); /** * delete tile from cache * * \memberof mapcache_cache */ - void (*tile_delete)(mapcache_context *ctx, mapcache_tile * tile); + void (*_tile_delete)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); - int (*tile_exists)(mapcache_context *ctx, mapcache_tile * tile); + int (*_tile_exists)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); /** * set tile content to cache * \memberof mapcache_cache */ - void (*tile_set)(mapcache_context *ctx, mapcache_tile * tile); - void (*tile_multi_set)(mapcache_context *ctx, mapcache_tile *tiles, int ntiles); + void (*_tile_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); + void (*_tile_multi_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles); void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_cache * cache, mapcache_cfg *config); void (*configuration_post_config)(mapcache_context *ctx, mapcache_cache * cache, mapcache_cfg *config); }; -/**\class mapcache_cache_disk - * \brief a mapcache_cache on a filesytem - * \implements mapcache_cache - */ -struct mapcache_cache_disk { - mapcache_cache cache; - char *base_directory; - char *filename_template; - int symlink_blank; - int creation_retry; - - /** - * Set filename for a given tile - * \memberof mapcache_cache_disk - */ - void (*tile_key)(mapcache_context *ctx, mapcache_tile *tile, char **path); -}; - -#ifdef USE_TIFF -struct mapcache_cache_tiff { - mapcache_cache cache; - char *filename_template; - char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; - int count_x; - int count_y; - mapcache_image_format_jpeg *format; -}; -#endif +MS_DLL_EXPORT int mapcache_cache_tile_get(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +void mapcache_cache_tile_delete(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +MS_DLL_EXPORT int mapcache_cache_tile_exists(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_cache_tile_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +void mapcache_cache_tile_multi_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles); + -#ifdef USE_SQLITE -/**\class mapcache_cache_sqlite - * \brief a mapcache_cache on a filesytem - * \implements mapcache_cache - */ -typedef struct mapcache_cache_sqlite mapcache_cache_sqlite; -typedef struct mapcache_cache_sqlite_stmt mapcache_cache_sqlite_stmt; - -struct mapcache_cache_sqlite_stmt { - char *sql; -}; - -struct mapcache_cache_sqlite { - mapcache_cache cache; - char *dbfile; - mapcache_cache_sqlite_stmt create_stmt; - mapcache_cache_sqlite_stmt exists_stmt; - mapcache_cache_sqlite_stmt get_stmt; - mapcache_cache_sqlite_stmt set_stmt; - mapcache_cache_sqlite_stmt delete_stmt; - apr_table_t *pragmas; - void (*bind_stmt)(mapcache_context*ctx, void *stmt, mapcache_tile *tile); - int n_prepared_statements; - int detect_blank; -}; /** * \memberof mapcache_cache_sqlite */ mapcache_cache* mapcache_cache_sqlite_create(mapcache_context *ctx); mapcache_cache* mapcache_cache_mbtiles_create(mapcache_context *ctx); -#endif -#ifdef USE_BDB -typedef struct mapcache_cache_bdb mapcache_cache_bdb; -struct mapcache_cache_bdb { - mapcache_cache cache; - char *basedir; - char *key_template; -}; +/** + * \memberof mapcache_cache_bdb + */ mapcache_cache *mapcache_cache_bdb_create(mapcache_context *ctx); -#endif - -#ifdef USE_TC -typedef struct mapcache_cache_tc mapcache_cache_tc; -struct mapcache_cache_tc { - mapcache_cache cache; - char *basedir; - char *key_template; - mapcache_context *ctx; -}; -mapcache_cache *mapcache_cache_tc_create(mapcache_context *ctx); -#endif - -#ifdef USE_MEMCACHE -typedef struct mapcache_cache_memcache mapcache_cache_memcache; -/**\class mapcache_cache_memcache - * \brief a mapcache_cache on memcached servers - * \implements mapcache_cache - */ -struct mapcache_cache_memcache { - mapcache_cache cache; - apr_memcache_t *memcache; -}; /** * \memberof mapcache_cache_memcache */ mapcache_cache* mapcache_cache_memcache_create(mapcache_context *ctx); -#endif + +/** + * \memberof mapcache_cache_couchbase + */ +mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx); + +/** + * \memberof mapcache_cache_tc + */ +mapcache_cache* mapcache_cache_tc_create(mapcache_context *ctx); + +/** + * \memberof mapcache_cache_riak + */ +mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx); /** @} */ @@ -542,8 +437,13 @@ mapcache_service *service; }; +struct mapcache_request_image { + mapcache_request request; + mapcache_image_format *format; +}; + struct mapcache_request_get_tile { - mapcache_request request; + mapcache_request_image image_request; /** * a list of tiles requested by the client @@ -557,8 +457,7 @@ * before being returned to the client */ int ntiles; - mapcache_image_format *format; - + int allow_redirect; }; struct mapcache_http_response { @@ -571,7 +470,7 @@ struct mapcache_map { mapcache_tileset *tileset; mapcache_grid_link *grid_link; - apr_table_t *dimensions; + apr_array_header_t *dimensions; mapcache_buffer *encoded_data; mapcache_image *raw_image; int nodata; /**< \sa mapcache_tile::nodata */ @@ -594,12 +493,11 @@ }; struct mapcache_request_get_map { - mapcache_request request; + mapcache_request_image image_request; mapcache_map **maps; int nmaps; mapcache_getmap_strategy getmap_strategy; mapcache_resample_mode resample_mode; - mapcache_image_format *getmap_format; }; struct mapcache_request_get_capabilities { @@ -616,49 +514,24 @@ char *mime_type; }; -struct mapcache_request_get_capabilities_tms { - mapcache_request_get_capabilities request; - mapcache_tileset *tileset; - mapcache_grid_link *grid_link; - char *version; -}; - -struct mapcache_request_get_capabilities_kml { - mapcache_request_get_capabilities request; - mapcache_tile *tile; - mapcache_tileset *tileset; - mapcache_grid_link *grid; -}; -struct mapcache_request_get_capabilities_wms { - mapcache_request_get_capabilities request; -}; - -struct mapcache_request_get_capabilities_wmts { - mapcache_request_get_capabilities request; -}; -/** - * the capabilities request for a specific service, to be able to create - * demo pages specific to a given service - */ -struct mapcache_request_get_capabilities_demo { - mapcache_request_get_capabilities request; - mapcache_service *service; +struct mapcache_forwarding_rule { + char *name; + mapcache_http *http; + apr_array_header_t *match_params; /* actually those are mapcache_dimensions */ + int append_pathinfo; + size_t max_post_len; }; struct mapcache_request_proxy { mapcache_request request; - mapcache_http *http; + mapcache_forwarding_rule *rule; apr_table_t *params; + apr_table_t *headers; const char *pathinfo; -}; - -struct mapcache_forwarding_rule { - char *name; - mapcache_http *http; - apr_array_header_t *match_params; /* actually those are mapcache_dimensions */ - int append_pathinfo; + char *post_buf; + size_t post_len; }; @@ -717,66 +590,9 @@ char **err_body, apr_table_t *headers); }; -/**\class mapcache_service_wms - * \brief an OGC WMS service - * \implements mapcache_service - */ -struct mapcache_service_wms { - mapcache_service service; - int maxsize; - apr_array_header_t *forwarding_rules; - mapcache_getmap_strategy getmap_strategy; - mapcache_resample_mode resample_mode; - mapcache_image_format *getmap_format; -}; - -/**\class mapcache_service_kml - * \brief a KML superoverlay service - * \implements mapcache_service - */ -struct mapcache_service_kml { - mapcache_service service; -}; - -/**\class mapcache_service_tms - * \brief a TMS service - * \implements mapcache_service - */ -struct mapcache_service_tms { - mapcache_service service; - int reverse_y; -}; -struct mapcache_service_mapguide { - mapcache_service service; - int rows_per_folder; - int cols_per_folder; -}; - -/**\class mapcache_service_wmts - * \brief a WMTS service - * \implements mapcache_service - */ -struct mapcache_service_wmts { - mapcache_service service; -}; - -/**\class mapcache_service_demo - * \brief a demo service - * \implements mapcache_service - */ -struct mapcache_service_demo { - mapcache_service service; -}; -/**\class mapcache_service_ve - * \brief a virtualearth service - * \implements mapcache_service - */ -struct mapcache_service_ve { - mapcache_service service; -}; /** * \brief create and initialize a mapcache_service_wms @@ -829,11 +645,15 @@ /** * \brief return the request that corresponds to the given url */ -void mapcache_service_dispatch_request(mapcache_context *ctx, +MS_DLL_EXPORT void mapcache_service_dispatch_request(mapcache_context *ctx, mapcache_request **request, char *pathinfo, apr_table_t *params, mapcache_cfg *config); +/** + * \brief return the number of enabled/configured services + */ +MS_DLL_EXPORT int mapcache_config_services_enabled(mapcache_context *ctx, mapcache_cfg *config); /** @} */ @@ -924,7 +744,9 @@ /** * \brief check if image has some non opaque pixels */ -int mapcache_image_has_alpha(mapcache_image *img); +int mapcache_image_has_alpha(mapcache_image *img, unsigned int cutoff); + +void mapcache_image_fill(mapcache_context *ctx, mapcache_image *image, const unsigned char *fill_color); /** @} */ @@ -932,19 +754,14 @@ /** \defgroup http HTTP Request handling*/ /** @{ */ void mapcache_http_do_request(mapcache_context *ctx, mapcache_http *req, mapcache_buffer *data, apr_table_t *headers, long *http_code); -void mapcache_http_do_request_with_params(mapcache_context *ctx, mapcache_http *req, apr_table_t *params, - mapcache_buffer *data, apr_table_t *headers, long *http_code); char* mapcache_http_build_url(mapcache_context *ctx, char *base, apr_table_t *params); -apr_table_t *mapcache_http_parse_param_string(mapcache_context *ctx, char *args); +MS_DLL_EXPORT apr_table_t *mapcache_http_parse_param_string(mapcache_context *ctx, char *args); /** @} */ /** \defgroup configuration Configuration*/ /** @{ */ -struct mapcache_server_cfg { - apr_hash_t *aliases; /**< list of mapcache configurations aliased to a server uri */ -}; @@ -954,12 +771,43 @@ MAPCACHE_MODE_MIRROR_SPLIT } mapcache_mode; +typedef enum { + MAPCACHE_LOCKER_DISK, + MAPCACHE_LOCKER_MEMCACHE, + MAPCACHE_LOCKER_FALLBACK +} mapcache_lock_mode; + +typedef enum { + MAPCACHE_LOCK_AQUIRED, + MAPCACHE_LOCK_LOCKED, + MAPCACHE_LOCK_NOENT +} mapcache_lock_result; + + +struct mapcache_locker{ + mapcache_lock_result (*aquire_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock); + mapcache_lock_result (*ping_lock)(mapcache_context *ctx, mapcache_locker *self, void *lock); + void (*release_lock)(mapcache_context *ctx, mapcache_locker *self, void *lock); + + void (*parse_xml)(mapcache_context *ctx, mapcache_locker *self, ezxml_t node); + mapcache_lock_mode type; + double timeout; + double retry_interval; /* time to wait before checking again on a lock, in seconds */ +}; + +mapcache_locker* mapcache_locker_disk_create(mapcache_context *ctx); + +mapcache_locker* mapcache_locker_memcache_create(mapcache_context *ctx); + +mapcache_locker* mapcache_locker_fallback_create(mapcache_context *ctx); + +void mapcache_config_parse_locker(mapcache_context *ctx, ezxml_t node, mapcache_locker **locker); +void mapcache_config_parse_locker_old(mapcache_context *ctx, ezxml_t doc, mapcache_cfg *config); + /** * a configuration that will be served */ struct mapcache_cfg { - char *configFile; /**< the filename from which this configuration was loaded */ - /** * a list of services that will be responded to */ @@ -1011,26 +859,10 @@ apr_table_t *metadata; - /** - * directory where lock files will be placed. - * Must be readable and writable by the apache user. - * Must be placed on a network mounted shared directory if multiple mapcache instances - * need to be synchronized - */ - const char *lockdir; - - /** - * time in nanoseconds to wait before rechecking for lockfile presence - */ - apr_interval_time_t lock_retry_interval; /* time in nanoseconds to wait before rechecking for lockfile presence */ + mapcache_locker *locker; int threaded_fetching; - /** - * the uri where the base of the service is mapped - */ - const char *endpoint; - /* for fastcgi only */ int autoreload; /* should the modification time of the config file be recorded and the file be reparsed if it is modified. */ @@ -1051,14 +883,14 @@ * @param pool * @return */ -void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi); -void mapcache_configuration_post_config(mapcache_context *ctx, mapcache_cfg *config); +MS_DLL_EXPORT void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi); +MS_DLL_EXPORT void mapcache_configuration_post_config(mapcache_context *ctx, mapcache_cfg *config); void mapcache_configuration_parse_xml(mapcache_context *ctx, const char *filename, mapcache_cfg *config); -mapcache_cfg* mapcache_configuration_create(apr_pool_t *pool); +MS_DLL_EXPORT mapcache_cfg* mapcache_configuration_create(apr_pool_t *pool); mapcache_source* mapcache_configuration_get_source(mapcache_cfg *config, const char *key); -mapcache_cache* mapcache_configuration_get_cache(mapcache_cfg *config, const char *key); +MS_DLL_EXPORT mapcache_cache* mapcache_configuration_get_cache(mapcache_cfg *config, const char *key); mapcache_grid *mapcache_configuration_get_grid(mapcache_cfg *config, const char *key); -mapcache_tileset* mapcache_configuration_get_tileset(mapcache_cfg *config, const char *key); +MS_DLL_EXPORT mapcache_tileset* mapcache_configuration_get_tileset(mapcache_cfg *config, const char *key); mapcache_image_format *mapcache_configuration_get_image_format(mapcache_cfg *config, const char *key); void mapcache_configuration_add_image_format(mapcache_cfg *config, mapcache_image_format *format, const char * key); void mapcache_configuration_add_source(mapcache_cfg *config, mapcache_source *source, const char * key); @@ -1073,21 +905,31 @@ void mapcache_source_init(mapcache_context *ctx, mapcache_source *source); /** + * \memberof mapcache_source + */ +void mapcache_source_render_map(mapcache_context *ctx, mapcache_source *source, mapcache_map *map); +void mapcache_source_query_info(mapcache_context *ctx, mapcache_source *source, + mapcache_feature_info *fi); + +/** * \memberof mapcache_source_gdal */ mapcache_source* mapcache_source_gdal_create(mapcache_context *ctx); /** + * \memberof mapcache_source_fallback + */ +mapcache_source* mapcache_source_fallback_create(mapcache_context *ctx); + +/** * \memberof mapcache_source_wms */ mapcache_source* mapcache_source_wms_create(mapcache_context *ctx); -#ifdef USE_MAPSERVER /** * \memberof mapcache_source_wms */ mapcache_source* mapcache_source_mapserver_create(mapcache_context *ctx); -#endif mapcache_source* mapcache_source_dummy_create(mapcache_context *ctx); @@ -1096,12 +938,22 @@ */ mapcache_cache* mapcache_cache_disk_create(mapcache_context *ctx); -#ifdef USE_TIFF +/** + * \memberof mapcache_cache_rest + */ +mapcache_cache* mapcache_cache_rest_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_s3_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_azure_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_google_create(mapcache_context *ctx); + /** * \memberof mapcache_cache_tiff */ mapcache_cache* mapcache_cache_tiff_create(mapcache_context *ctx); -#endif + +mapcache_cache* mapcache_cache_composite_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_fallback_create(mapcache_context *ctx); +mapcache_cache* mapcache_cache_multitier_create(mapcache_context *ctx); /** \defgroup tileset Tilesets*/ @@ -1125,11 +977,14 @@ * \sa mapcache_image_format */ mapcache_buffer *encoded_data; + char *redirect; + int allow_redirect; mapcache_image *raw_image; apr_time_t mtime; /**< last modification time */ int expires; /**< time in seconds after which the tile should be rechecked for validity */ - apr_table_t *dimensions; + apr_array_header_t *dimensions; + /** * flag stating the tile is empty (i.e. fully transparent). * if set, this indicates that there was no error per se, but that there was @@ -1168,7 +1023,7 @@ MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT, MAPCACHE_GRID_ORIGIN_TOP_LEFT, MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT, - MAPCACHE_GRID_ORIGIN_TOP_RIGHT, + MAPCACHE_GRID_ORIGIN_TOP_RIGHT } mapcache_grid_origin; struct mapcache_grid { @@ -1200,7 +1055,7 @@ mapcache_extent *restricted_extent; mapcache_extent_i *grid_limits; int minz,maxz; - + /** * tiles above this zoom level will not be stored to the cache, but will be * dynamically generated (either by reconstructing from lower level tiles, or @@ -1209,6 +1064,8 @@ int max_cached_zoom; mapcache_outofzoom_strategy outofzoom_strategy; + + apr_array_header_t *intermediate_grids; }; /**\class mapcache_tileset @@ -1258,11 +1115,12 @@ int auto_expire; int read_only; + int subdimension_read_only; /** * the cache in which the tiles should be stored */ - mapcache_cache *cache; + mapcache_cache *_cache; /** * the source from which tiles should be requested @@ -1279,7 +1137,9 @@ */ apr_array_header_t *dimensions; - mapcache_timedimension *timedimension; + int store_dimension_assemblies;/**< should multiple sub-dimensions be assembled dynamically (per-request) or should they be cached once assembled */ + + mapcache_dimension_assembly_type dimension_assembly_type; /** * image to be used as a watermark @@ -1301,7 +1161,8 @@ mapcache_grid_link *grid_link, mapcache_extent *bbox, int width, int height, int *ntiles, - mapcache_tile ***tiles); + mapcache_tile ***tiles, + mapcache_grid_link **effectively_used_grid_link); mapcache_image* mapcache_tileset_assemble_map_tiles(mapcache_context *ctx, mapcache_tileset *tileset, mapcache_grid_link *grid_link, @@ -1325,6 +1186,9 @@ * @return */ void mapcache_tileset_tile_validate(mapcache_context *ctx, mapcache_tile *tile); +void mapcache_tileset_tile_validate_z(mapcache_context *ctx, mapcache_tile *tile); +void mapcache_tileset_tile_validate_x(mapcache_context *ctx, mapcache_tile *tile); +void mapcache_tileset_tile_validate_y(mapcache_context *ctx, mapcache_tile *tile); /** * compute level for a given resolution @@ -1338,14 +1202,15 @@ */ void mapcache_tileset_get_level(mapcache_context *ctx, mapcache_tileset *tileset, double *resolution, int *level); -void mapcache_grid_get_closest_level(mapcache_context *ctx, mapcache_grid_link *grid, double resolution, int *level); -void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile); +mapcache_grid_link* mapcache_grid_get_closest_wms_level(mapcache_context *ctx, mapcache_grid_link *grid, double resolution, int *level); +MS_DLL_EXPORT void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile); /** * \brief delete tile from cache * @param whole_metatile delete all the other tiles from the metatile to */ -void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile); +MS_DLL_EXPORT void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile); int mapcache_grid_is_bbox_aligned(mapcache_context *ctx, mapcache_grid *grid, mapcache_extent *bbox); @@ -1356,7 +1221,7 @@ * @param pool * @return */ -mapcache_tile* mapcache_tileset_tile_create(apr_pool_t *pool, mapcache_tileset *tileset, mapcache_grid_link *grid_link); +MS_DLL_EXPORT mapcache_tile* mapcache_tileset_tile_create(apr_pool_t *pool, mapcache_tileset *tileset, mapcache_grid_link *grid_link); mapcache_tile* mapcache_tileset_tile_clone(apr_pool_t *pool, mapcache_tile *src); @@ -1389,27 +1254,27 @@ void mapcache_tileset_add_watermark(mapcache_context *ctx, mapcache_tileset *tileset, const char *filename); -int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, char *resource); -void mapcache_unlock_resource(mapcache_context *ctx, char *resource); +MS_DLL_EXPORT int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void **lock); +MS_DLL_EXPORT void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, void *lock); -mapcache_metatile* mapcache_tileset_metatile_get(mapcache_context *ctx, mapcache_tile *tile); -void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt); -char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt); +MS_DLL_EXPORT mapcache_metatile* mapcache_tileset_metatile_get(mapcache_context *ctx, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt); +MS_DLL_EXPORT char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt); /** @} */ -mapcache_http_response* mapcache_core_get_capabilities(mapcache_context *ctx, mapcache_service *service, mapcache_request_get_capabilities *req_caps, char *url, char *path_info, mapcache_cfg *config); -mapcache_http_response* mapcache_core_get_tile(mapcache_context *ctx, mapcache_request_get_tile *req_tile); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_capabilities(mapcache_context *ctx, mapcache_service *service, mapcache_request_get_capabilities *req_caps, char *url, char *path_info, mapcache_cfg *config); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_tile(mapcache_context *ctx, mapcache_request_get_tile *req_tile); -mapcache_http_response* mapcache_core_get_map(mapcache_context *ctx, mapcache_request_get_map *req_map); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_map(mapcache_context *ctx, mapcache_request_get_map *req_map); -mapcache_http_response* mapcache_core_get_featureinfo(mapcache_context *ctx, mapcache_request_get_feature_info *req_fi); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_featureinfo(mapcache_context *ctx, mapcache_request_get_feature_info *req_fi); -mapcache_http_response* mapcache_core_proxy_request(mapcache_context *ctx, mapcache_request_proxy *req_proxy); -mapcache_http_response* mapcache_core_respond_to_error(mapcache_context *ctx); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_proxy_request(mapcache_context *ctx, mapcache_request_proxy *req_proxy); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_respond_to_error(mapcache_context *ctx); /* in grid.c */ @@ -1418,8 +1283,11 @@ const char* mapcache_grid_get_crs(mapcache_context *ctx, mapcache_grid *grid); const char* mapcache_grid_get_srs(mapcache_context *ctx, mapcache_grid *grid); -void mapcache_grid_get_extent(mapcache_context *ctx, mapcache_grid *grid, +MS_DLL_EXPORT void mapcache_grid_get_tile_extent(mapcache_context *ctx, mapcache_grid *grid, int x, int y, int z, mapcache_extent *bbox); + +MS_DLL_EXPORT void mapcache_grid_get_metatile_extent(mapcache_context *ctx, mapcache_tile *tile, mapcache_extent *bbox); + /** * \brief compute x y value for given lon/lat (dx/dy) and given zoomlevel * @param ctx @@ -1430,7 +1298,7 @@ * @param x * @param y */ -void mapcache_grid_get_xy(mapcache_context *ctx, mapcache_grid *grid, double dx, double dy, int z, int *x, int *y); +MS_DLL_EXPORT void mapcache_grid_get_xy(mapcache_context *ctx, mapcache_grid *grid, double dx, double dy, int z, int *x, int *y); double mapcache_grid_get_resolution(mapcache_extent *bbox, int sx, int sy); double mapcache_grid_get_horizontal_resolution(mapcache_extent *bbox, int width); @@ -1450,15 +1318,18 @@ * \param extent * \param tolerance the number of tiles around the given extent that can be requested without returning an error. */ -void mapcache_grid_compute_limits(const mapcache_grid *grid, const mapcache_extent *extent, mapcache_extent_i *limits, int tolerance); +MS_DLL_EXPORT void mapcache_grid_compute_limits(const mapcache_grid *grid, const mapcache_extent *extent, mapcache_extent_i *limits, int tolerance); /* in util.c */ -int mapcache_util_extract_int_list(mapcache_context *ctx, const char* args, const char *sep, int **numbers, +MS_DLL_EXPORT int mapcache_util_extract_int_list(mapcache_context *ctx, const char* args, const char *sep, int **numbers, int *numbers_count); -int mapcache_util_extract_double_list(mapcache_context *ctx, const char* args, const char *sep, double **numbers, +MS_DLL_EXPORT int mapcache_util_extract_double_list(mapcache_context *ctx, const char* args, const char *sep, double **numbers, int *numbers_count); char *mapcache_util_str_replace(apr_pool_t *pool, const char *string, const char *substr, const char *replacement ); +void mapcache_util_quadkey_decode(mapcache_context *ctx, const char *quadkey, int *x, int *y, int *z); + +char* mapcache_util_quadkey_encode(mapcache_context *ctx, int x, int y, int z); /** * \brief replace dangerous characters in string @@ -1470,10 +1341,19 @@ */ char* mapcache_util_str_sanitize(apr_pool_t *pool, const char *str, const char* from, char to); +typedef enum { + MAPCACHE_UTIL_XML_SECTION_TEXT, + MAPCACHE_UTIL_XML_SECTION_ATTRIBUTE, + MAPCACHE_UTIL_XML_SECTION_COMMENT +} mapcache_util_xml_section_type; + +char* mapcache_util_str_xml_escape(apr_pool_t *pool, const char *str, mapcache_util_xml_section_type xml_section_type); + char* mapcache_util_get_tile_dimkey(mapcache_context *ctx, mapcache_tile *tile, char* sanitized_chars, char *sanitize_to); char* mapcache_util_get_tile_key(mapcache_context *ctx, mapcache_tile *tile, char *stemplate, char* sanitized_chars, char *sanitize_to); +void mapcache_make_parent_dirs(mapcache_context *ctx, char *filename); /**\defgroup imageio Image IO */ /** @{ */ @@ -1484,6 +1364,7 @@ typedef enum { MAPCACHE_COMPRESSION_BEST, /**< best but slowest compression*/ MAPCACHE_COMPRESSION_FAST, /**< fast compression*/ + MAPCACHE_COMPRESSION_DISABLE, /**< no compression*/ MAPCACHE_COMPRESSION_DEFAULT /**< default compression*/ } mapcache_compression_type; @@ -1495,6 +1376,15 @@ MAPCACHE_PHOTOMETRIC_YCBCR } mapcache_photometric; +/** + * optimization settings (mostly) for jpeg + */ +typedef enum { + MAPCACHE_OPTIMIZE_NO, + MAPCACHE_OPTIMIZE_YES, + MAPCACHE_OPTIMIZE_ARITHMETIC +} mapcache_optimization; + /**\interface mapcache_image_format * \brief an image format * \sa mapcache_image_format_jpeg @@ -1533,13 +1423,14 @@ mapcache_image_format format; mapcache_image_format *transparent; mapcache_image_format *opaque; + unsigned int alpha_cutoff; /* default 255. pixels with alpha >= alpha_cutoff will be considered fully opaque */ }; -mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, const unsigned char *hex_color, int *is_empty); +mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, int width, int height, const unsigned char *hex_color, int *is_empty); mapcache_image_format* mapcache_imageio_create_mixed_format(apr_pool_t *pool, - char *name, mapcache_image_format *transparent, mapcache_image_format *opaque); + char *name, mapcache_image_format *transparent, mapcache_image_format *opaque, unsigned int alpha_cutoff); /**\class mapcache_image_format_png_q * \brief Quantized PNG format @@ -1602,10 +1493,11 @@ mapcache_image_format format; int quality; /**< JPEG quality, 1-100 */ mapcache_photometric photometric; + mapcache_optimization optimize; }; mapcache_image_format* mapcache_imageio_create_jpeg_format(apr_pool_t *pool, char *name, int quality, - mapcache_photometric photometric); + mapcache_photometric photometric, mapcache_optimization optimize); /** * @param r @@ -1654,12 +1546,19 @@ double resolution; } mapcache_interval; -typedef enum { - MAPCACHE_DIMENSION_VALUES, - MAPCACHE_DIMENSION_REGEX, - MAPCACHE_DIMENSION_INTERVALS, - MAPCACHE_DIMENSION_TIME -} mapcache_dimension_type; +struct mapcache_requested_dimension { + mapcache_dimension *dimension; + char *requested_value; + char *cached_value; +}; + +void mapcache_tile_set_cached_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value); +void mapcache_map_set_cached_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value); +void mapcache_tile_set_requested_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value); +void mapcache_map_set_requested_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value); +void mapcache_set_requested_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value); +void mapcache_set_cached_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value); +MS_DLL_EXPORT apr_array_header_t *mapcache_requested_dimensions_clone(apr_pool_t *pool, apr_array_header_t *src); struct mapcache_dimension { mapcache_dimension_type type; @@ -1669,21 +1568,22 @@ char *default_value; /** - * \brief validate the given value - * - * \param value is updated in case the given value is correct but has to be represented otherwise, - * e.g. to round off a value - * \returns MAPCACHE_SUCCESS if the given value is correct for the current dimension - * \returns MAPCACHE_FAILURE if not + * \brief return the list of dimension values that match the requested entry */ - int (*validate)(mapcache_context *context, mapcache_dimension *dimension, char **value); + apr_array_header_t* (*get_entries_for_value)(mapcache_context *ctx, mapcache_dimension *dimension, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid); /** - * \brief returns a list of values that are authorized for this dimension - * - * \returns a list of character strings that will be included in the capabilities element + * \brief return all possible values */ - const char** (*print_ogc_formatted_values)(mapcache_context *context, mapcache_dimension *dimension); + apr_array_header_t* (*get_all_entries)(mapcache_context *ctx, mapcache_dimension *dimension, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid); + + /** + * \brief return all possible values formatted in a way compatible with OGC capabilities element + */ + apr_array_header_t* (*get_all_ogc_formatted_entries)(mapcache_context *ctx, mapcache_dimension *dimension, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid); /** * \brief parse the value given in the configuration @@ -1691,74 +1591,33 @@ void (*configuration_parse_xml)(mapcache_context *context, mapcache_dimension *dim, ezxml_t node); }; -struct mapcache_dimension_values { - mapcache_dimension dimension; - int nvalues; - char **values; - int case_sensitive; -}; - -struct mapcache_dimension_regex { - mapcache_dimension dimension; - char *regex_string; -#ifdef USE_PCRE - pcre *pcregex; -#else - regex_t *regex; -#endif -}; - -struct mapcache_dimension_intervals { - mapcache_dimension dimension; - int nintervals; - mapcache_interval *intervals; -}; - -struct mapcache_dimension_time { - mapcache_dimension dimension; - int nintervals; - mapcache_interval *intervals; -}; - -mapcache_dimension* mapcache_dimension_values_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_regex_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_intervals_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_time_create(apr_pool_t *pool); - -typedef enum { - MAPCACHE_TIMEDIMENSION_ASSEMBLY_STACK, - MAPCACHE_TIMEDIMENSION_ASSEMBLY_ANIMATE -} mapcache_timedimension_assembly_type; - -typedef enum { - MAPCACHE_TIMEDIMENSION_SOURCE_SQLITE -} mapcache_timedimension_source_type; - -apr_array_header_t* mapcache_timedimension_get_entries_for_value(mapcache_context *ctx, mapcache_timedimension *timedimesnion, - mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, const char *value); - -struct mapcache_timedimension { - mapcache_timedimension_assembly_type assembly_type; - void (*configuration_parse_xml)(mapcache_context *context, mapcache_timedimension *dim, ezxml_t node); - apr_array_header_t* (*get_entries_for_interval)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset, - mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end); - apr_array_header_t* (*get_all_entries)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset); - char *default_value; - char *key; /* TIME, hardcoded */ -}; - -#ifdef USE_SQLITE -typedef struct mapcache_timedimension_sqlite mapcache_timedimension_sqlite; -struct mapcache_timedimension_sqlite { - mapcache_timedimension timedimension; - char *dbfile; - char *query; -}; -mapcache_timedimension* mapcache_timedimension_sqlite_create(apr_pool_t *pool); -#endif +mapcache_dimension* mapcache_dimension_values_create(mapcache_context *ctx, apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_sqlite_create(mapcache_context *ctx, apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_regex_create(mapcache_context *ctx, apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_time_create(mapcache_context *ctx, apr_pool_t *pool); int mapcache_is_axis_inverted(const char *srs); +typedef struct mapcache_pooled_connection_container mapcache_pooled_connection_container; +typedef struct mapcache_pooled_connection mapcache_pooled_connection; +typedef struct mapcache_pooled_connection_private_data mapcache_pooled_connection_private_data; + +struct mapcache_pooled_connection { + mapcache_pooled_connection_private_data *private; + void *connection; +}; + +typedef void (*mapcache_connection_constructor)(mapcache_context *ctx, void **connection, void *params); +typedef void (*mapcache_connection_destructor)(void *connection); + +MS_DLL_EXPORT apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool); +mapcache_pooled_connection* mapcache_connection_pool_get_connection(mapcache_context *ctx, char *key, + mapcache_connection_constructor constructor, + mapcache_connection_destructor destructor, + void *params); +void mapcache_connection_pool_invalidate_connection(mapcache_context *ctx, mapcache_pooled_connection *connection); +void mapcache_connection_pool_release_connection(mapcache_context *ctx, mapcache_pooled_connection *connection); + #endif /* MAPCACHE_H_ */ /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/include/mapcache_services.h mapcache-1.6.1/include/mapcache_services.h --- mapcache-1.2.1/include/mapcache_services.h 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/include/mapcache_services.h 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,162 @@ +/****************************************************************************** + * + * Project: MapCache + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2016 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +/*! \file mapcache_services.h + \brief structure declarations for supported services + */ + + +#ifndef MAPCACHE_SERVICES_H_ +#define MAPCACHE_SERVICES_H_ + +#include "mapcache.h" + + +/** \addtogroup services */ +/** @{ */ + +typedef struct mapcache_request_get_capabilities_wmts mapcache_request_get_capabilities_wmts; +typedef struct mapcache_service_wmts mapcache_service_wmts; + +struct mapcache_request_get_capabilities_wmts { + mapcache_request_get_capabilities request; +}; + +/**\class mapcache_service_wmts + * \brief a WMTS service + * \implements mapcache_service + */ +struct mapcache_service_wmts { + mapcache_service service; +}; + + + +typedef struct mapcache_service_wms mapcache_service_wms; +typedef struct mapcache_request_get_capabilities_wms mapcache_request_get_capabilities_wms; + +struct mapcache_request_get_capabilities_wms { + mapcache_request_get_capabilities request; +}; + +/**\class mapcache_service_wms + * \brief an OGC WMS service + * \implements mapcache_service + */ +struct mapcache_service_wms { + mapcache_service service; + int maxsize; + apr_array_header_t *forwarding_rules; + mapcache_getmap_strategy getmap_strategy; + mapcache_resample_mode resample_mode; + mapcache_image_format *getmap_format; + int allow_format_override; /* can the client specify which image format should be returned */ +}; + +typedef struct mapcache_service_ve mapcache_service_ve; + +/**\class mapcache_service_ve + * \brief a virtualearth service + * \implements mapcache_service + */ +struct mapcache_service_ve { + mapcache_service service; +}; + + +typedef struct mapcache_service_gmaps mapcache_service_gmaps; +typedef struct mapcache_service_tms mapcache_service_tms; +typedef struct mapcache_request_get_capabilities_tms mapcache_request_get_capabilities_tms; + +/**\class mapcache_service_tms + * \brief a TMS service + * \implements mapcache_service + */ +struct mapcache_service_tms { + mapcache_service service; + int reverse_y; +}; + +struct mapcache_request_get_capabilities_tms { + mapcache_request_get_capabilities request; + mapcache_tileset *tileset; + mapcache_grid_link *grid_link; + char *version; +}; + + +typedef struct mapcache_service_mapguide mapcache_service_mapguide; + +struct mapcache_service_mapguide { + mapcache_service service; + int rows_per_folder; + int cols_per_folder; +}; + + + +typedef struct mapcache_service_kml mapcache_service_kml; +typedef struct mapcache_request_get_capabilities_kml mapcache_request_get_capabilities_kml; +struct mapcache_request_get_capabilities_kml { + mapcache_request_get_capabilities request; + mapcache_tile *tile; + mapcache_tileset *tileset; + mapcache_grid_link *grid; +}; +/**\class mapcache_service_kml + * \brief a KML superoverlay service + * \implements mapcache_service + */ +struct mapcache_service_kml { + mapcache_service service; +}; + +typedef struct mapcache_request_get_capabilities_demo mapcache_request_get_capabilities_demo; +typedef struct mapcache_service_demo mapcache_service_demo; + +/** + * the capabilities request for a specific service, to be able to create + * demo pages specific to a given service + */ +struct mapcache_request_get_capabilities_demo { + mapcache_request_get_capabilities request; + mapcache_service *service; +}; + +/**\class mapcache_service_demo + * \brief a demo service + * \implements mapcache_service + */ +struct mapcache_service_demo { + mapcache_service service; + +}; + + + + +#endif /*MAPCACHE_SERVICES_H*/ diff -Nru mapcache-1.2.1/include/util.h mapcache-1.6.1/include/util.h --- mapcache-1.2.1/include/util.h 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/include/util.h 2017-09-29 21:31:13.000000000 +0000 @@ -87,13 +87,19 @@ #endif #endif +#if defined(_WIN32) && !defined(__CYGWIN__) +# define MS_DLL_EXPORT __declspec(dllexport) +#else +#define MS_DLL_EXPORT +#endif #if defined(_WIN32) struct mctimeval { long tv_sec; /* seconds */ long tv_usec; /* and microseconds */ }; -void mapcache_gettimeofday(struct mctimeval *t, void *__not_used_here__); +MS_DLL_EXPORT void mapcache_gettimeofday(struct mctimeval *t, void *__not_used_here__); +MS_DLL_EXPORT char * strptime (const char *buf, const char *format, struct tm *timeptr); #else # include /* for gettimeofday() */ # define mctimeval timeval diff -Nru mapcache-1.2.1/lib/cache_bdb.c mapcache-1.6.1/lib/cache_bdb.c --- mapcache-1.2.1/lib/cache_bdb.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/cache_bdb.c 2017-09-29 21:31:13.000000000 +0000 @@ -1,7 +1,6 @@ /****************************************************************************** - * $Id$ * - * Project: MapServer + * Project: MapCache * Purpose: MapCache tile caching support file: Berkeley DB cache backend * Author: Thomas Bonfort and the MapServer team. * @@ -27,10 +26,8 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" -#ifdef USE_BDB - #include "mapcache.h" +#ifdef USE_BDB #include #include #include @@ -51,8 +48,12 @@ #define PAGESIZE 64*1024 #define CACHESIZE 1024*1024 -static apr_hash_t *ro_connection_pools = NULL; -static apr_hash_t *rw_connection_pools = NULL; +typedef struct mapcache_cache_bdb mapcache_cache_bdb; +struct mapcache_cache_bdb { + mapcache_cache cache; + char *basedir; + char *key_template; +}; struct bdb_env { DB* db; @@ -61,168 +62,100 @@ char *errmsg; }; -static apr_status_t _bdb_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) +void mapcache_bdb_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; int env_flags; int mode; mapcache_cache_bdb *cache = (mapcache_cache_bdb*)params; - char *dbfile = apr_pstrcat(pool,cache->basedir,"/",cache->cache.name,".db",NULL); + char *dbfile = malloc(strlen(cache->basedir)+strlen(cache->cache.name)+strlen("/.db")+1); struct bdb_env *benv = calloc(1,sizeof(struct bdb_env)); + dbfile[0]=0; + strcat(strcat(strcat(strcat(dbfile,cache->basedir),"/"),cache->cache.name),".db"); /*yuk yuk*/ *conn_ = benv; ret = db_env_create(&benv->env, 0); if(ret) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for db_env_create: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx, 500, "bdb cache failure for db_env_create: %s", db_strerror(ret)); + goto cleanup_error; } ret = benv->env->set_cachesize(benv->env,0,CACHESIZE,1); /* set a larger cache size than default */ if(ret) { - benv->errmsg = apr_psprintf(pool, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx, 500, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret)); + goto cleanup_error; } env_flags = DB_INIT_CDB|DB_INIT_MPOOL|DB_CREATE; ret = benv->env->open(benv->env,cache->basedir,env_flags,0); if(ret) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for env->open: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure for env->open: %s", db_strerror(ret)); + goto cleanup_error; } if ((ret = db_create(&benv->db, benv->env, 0)) != 0) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for db_create: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure for db_create: %s", db_strerror(ret)); + goto cleanup_error; } mode = DB_BTREE; ret = benv->db->set_pagesize(benv->db,PAGESIZE); /* set pagesize to maximum allowed, as tile data is usually pretty large */ if(ret) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret)); + goto cleanup_error; } if ((ret = benv->db->open(benv->db, NULL, dbfile, NULL, mode, DB_CREATE, 0664)) != 0) { - benv->errmsg = apr_psprintf(pool,"bdb cache failure 1 for db->open: %s", db_strerror(ret)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"bdb cache failure 1 for db->open: %s", db_strerror(ret)); + goto cleanup_error; } - return APR_SUCCESS; + + goto cleanup; + +cleanup_error: + free(benv); +cleanup: + free(dbfile); } -static apr_status_t _bdb_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) +void mapcache_bdb_connection_destructor(void *conn_) { struct bdb_env *benv = (struct bdb_env*)conn_; benv->db->close(benv->db,0); benv->env->close(benv->env,0); free(benv); - - return APR_SUCCESS; } -static struct bdb_env* _bdb_get_conn(mapcache_context *ctx, mapcache_tile* tile, int readonly) { - apr_status_t rv; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; - +static mapcache_pooled_connection* _bdb_get_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile* tile, int readonly) { struct bdb_env *benv; - apr_hash_t *pool_container; - apr_reslist_t *pool = NULL; - if(readonly) { - pool_container = ro_connection_pools; - } else { - pool_container = rw_connection_pools; - } - if(!pool_container || NULL == (pool = apr_hash_get(pool_container,cache->cache.name, APR_HASH_KEY_STRING)) ) { -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(!ro_connection_pools) { - ro_connection_pools = apr_hash_make(ctx->process_pool); - rw_connection_pools = apr_hash_make(ctx->process_pool); - } - - /* probably doesn't exist, unless the previous mutex locked us, so we check */ - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - if(!pool) { - /* there where no existing connection pools, create them*/ - rv = apr_reslist_create(&pool, - 0 /* min */, - 10 /* soft max */, - 200 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _bdb_reslist_get_connection, /* resource constructor */ - _bdb_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb ro connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(ro_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - rv = apr_reslist_create(&pool, - 0 /* min */, - 1 /* soft max */, - 1 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _bdb_reslist_get_connection, /* resource constructor */ - _bdb_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb rw connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(rw_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - } -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(readonly) - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - else - pool = apr_hash_get(rw_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - assert(pool); - } - rv = apr_reslist_acquire(pool, (void **)&benv); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to aquire connection to bdb backend: %s", (benv&& benv->errmsg)?benv->errmsg:"unknown error"); - return NULL; - } + mapcache_pooled_connection *pc; + char *conn_key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",cache->cache.name,NULL); + pc = mapcache_connection_pool_get_connection(ctx,conn_key,mapcache_bdb_connection_constructor, mapcache_bdb_connection_destructor, cache); + if(GC_HAS_ERROR(ctx)) return NULL; + benv = pc->connection; benv->readonly = readonly; - return benv; + return pc; } -static void _bdb_release_conn(mapcache_context *ctx, mapcache_tile *tile, struct bdb_env *benv) +static void _bdb_release_conn(mapcache_context *ctx, mapcache_cache_bdb *cache, mapcache_tile *tile, mapcache_pooled_connection *pc) { - apr_reslist_t *pool; - apr_hash_t *pool_container; - if(benv->readonly) { - pool_container = ro_connection_pools; - } else { - pool_container = rw_connection_pools; - } - pool = apr_hash_get(pool_container,tile->tileset->cache->name, APR_HASH_KEY_STRING); if(GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(pool,(void*)benv); + mapcache_connection_pool_invalidate_connection(ctx, pc); } else { - apr_reslist_release(pool, (void*)benv); + mapcache_connection_pool_release_connection(ctx,pc); } } -static int _mapcache_cache_bdb_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_bdb_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { int ret; DBT key; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - struct bdb_env *benv = _bdb_get_conn(ctx,tile,1); + mapcache_pooled_connection *pc; + struct bdb_env *benv; + pc = _bdb_get_conn(ctx,cache,tile,1); if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; + benv = pc->connection; memset(&key, 0, sizeof(DBT)); key.data = skey; key.size = strlen(skey)+1; @@ -237,18 +170,21 @@ ctx->set_error(ctx,500,"bdb backend failure on tile_exists: %s",db_strerror(ret)); ret= MAPCACHE_FALSE; } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); return ret; } -static void _mapcache_cache_bdb_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_bdb_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { DBT key; int ret; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - struct bdb_env *benv = _bdb_get_conn(ctx,tile,0); + mapcache_pooled_connection *pc; + struct bdb_env *benv; + pc = _bdb_get_conn(ctx,cache,tile,0); GC_CHECK_ERROR(ctx); + benv = pc->connection; memset(&key, 0, sizeof(DBT)); key.data = skey; key.size = strlen(skey)+1; @@ -260,17 +196,21 @@ if(ret) ctx->set_error(ctx,500,"bdb backend sync failure on tile_delete: %s",db_strerror(ret)); } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); } -static int _mapcache_cache_bdb_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_bdb_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { DBT key,data; int ret; - struct bdb_env *benv = _bdb_get_conn(ctx,tile,1); - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; - char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - if(GC_HAS_ERROR(ctx)) return MAPCACHE_FAILURE; + char *skey; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; + mapcache_pooled_connection *pc; + struct bdb_env *benv; + pc = _bdb_get_conn(ctx,cache,tile,1); + if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; + benv = pc->connection; + skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); data.flags = DB_DBT_MALLOC; @@ -282,7 +222,7 @@ if(ret == 0) { if(((char*)(data.data))[0] == '#') { - tile->encoded_data = mapcache_empty_png_decode(ctx,(unsigned char*)data.data,&tile->nodata); + tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy, (unsigned char*)data.data,&tile->nodata); } else { tile->encoded_data = mapcache_buffer_create(0,ctx->pool); tile->encoded_data->buf = data.data; @@ -298,20 +238,20 @@ ctx->set_error(ctx,500,"bdb backend failure on tile_get: %s",db_strerror(ret)); ret = MAPCACHE_FAILURE; } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); return ret; } -static void _mapcache_cache_bdb_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_bdb_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { DBT key,data; int ret; apr_time_t now; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tile->tileset->cache; + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); - struct bdb_env *benv = _bdb_get_conn(ctx,tile,0); - GC_CHECK_ERROR(ctx); + mapcache_pooled_connection *pc; + struct bdb_env *benv; now = apr_time_now(); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); @@ -323,6 +263,7 @@ tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); } + if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { data.size = 5+sizeof(apr_time_t); data.data = apr_palloc(ctx->pool,data.size); @@ -340,6 +281,11 @@ tile->encoded_data->size -= sizeof(apr_time_t); } + pc = _bdb_get_conn(ctx,cache,tile,0); + GC_CHECK_ERROR(ctx); + benv = pc->connection; + + ret = benv->db->put(benv->db,NULL,&key,&data,0); if(ret != 0) { ctx->set_error(ctx,500,"dbd backend failed on tile_set: %s", db_strerror(ret)); @@ -348,21 +294,25 @@ if(ret) ctx->set_error(ctx,500,"bdb backend sync failure on tile_set: %s",db_strerror(ret)); } - _bdb_release_conn(ctx,tile,benv); + _bdb_release_conn(ctx,cache,tile,pc); } -static void _mapcache_cache_bdb_multiset(mapcache_context *ctx, mapcache_tile *tiles, int ntiles) +static void _mapcache_cache_bdb_multiset(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { DBT key,data; int ret,i; apr_time_t now; - mapcache_cache_bdb *cache = (mapcache_cache_bdb*)tiles[0].tileset->cache; - struct bdb_env *benv = _bdb_get_conn(ctx,&tiles[0],0); - GC_CHECK_ERROR(ctx); + mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; + mapcache_pooled_connection *pc; + struct bdb_env *benv; now = apr_time_now(); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); + pc = _bdb_get_conn(ctx,cache,&tiles[0],0); + GC_CHECK_ERROR(ctx); + benv = pc->connection; + for(i=0; ikey_template,NULL,NULL); if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); - GC_CHECK_ERROR(ctx); + if(GC_HAS_ERROR(ctx)) { + _bdb_release_conn(ctx,cache,&tiles[0],pc); + return; + } } if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { data.size = 5+sizeof(apr_time_t); @@ -383,7 +336,10 @@ } else { if(!tile->encoded_data) { tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); - GC_CHECK_ERROR(ctx); + if(GC_HAS_ERROR(ctx)) { + _bdb_release_conn(ctx,cache,&tiles[0],pc); + return; + } } mapcache_buffer_append(tile->encoded_data,sizeof(apr_time_t),&now); data.data = tile->encoded_data->buf; @@ -404,7 +360,7 @@ if(ret) ctx->set_error(ctx,500,"bdb backend sync failure on sync in tile_multiset: %s",db_strerror(ret)); } - _bdb_release_conn(ctx,&tiles[0],benv); + _bdb_release_conn(ctx,cache,&tiles[0],pc); } @@ -454,11 +410,11 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_BDB; - cache->cache.tile_delete = _mapcache_cache_bdb_delete; - cache->cache.tile_get = _mapcache_cache_bdb_get; - cache->cache.tile_exists = _mapcache_cache_bdb_has_tile; - cache->cache.tile_set = _mapcache_cache_bdb_set; - cache->cache.tile_multi_set = _mapcache_cache_bdb_multiset; + cache->cache._tile_delete = _mapcache_cache_bdb_delete; + cache->cache._tile_get = _mapcache_cache_bdb_get; + cache->cache._tile_exists = _mapcache_cache_bdb_has_tile; + cache->cache._tile_set = _mapcache_cache_bdb_set; + cache->cache._tile_multi_set = _mapcache_cache_bdb_multiset; cache->cache.configuration_post_config = _mapcache_cache_bdb_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_bdb_configuration_parse_xml; cache->basedir = NULL; @@ -466,6 +422,11 @@ return (mapcache_cache*)cache; } +#else +mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"BERKELEYDB support not compiled in this version"); + return NULL; +} #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.2.1/lib/cache.c mapcache-1.6.1/lib/cache.c --- mapcache-1.2.1/lib/cache.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,159 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: generic cache access + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2015 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without compositeriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ +#include "mapcache.h" +#include + +int mapcache_cache_tile_get(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i,rv; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_get on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) get retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_get(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } + return rv; +} + +void mapcache_cache_tile_delete(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_delete on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + if(tile->tileset->read_only) + return; + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) delete retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_delete(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } +} + +int mapcache_cache_tile_exists(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i,rv; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_exists on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) exists retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_exists(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } + return rv; +} + +void mapcache_cache_tile_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_set on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + if(tile->tileset->read_only) + return; + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) set retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_set(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } +} + +void mapcache_cache_tile_multi_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles) { + int i; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_multi_set on cache (%s): (tileset=%s, grid=%s, first tile: z=%d, x=%d, y=%d",cache->name,tiles[0].tileset->name,tiles[0].grid_link->grid->name, + tiles[0].z,tiles[0].x, tiles[0].y); +#endif + if((&tiles[0])->tileset->read_only) + return; + if(cache->_tile_multi_set) { + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) multi-set retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_multi_set(ctx,cache,tiles,ntiles); + if(!GC_HAS_ERROR(ctx)) + break; + } + } else { + for( i=0;i + +typedef struct mapcache_cache_composite mapcache_cache_composite; + +typedef struct mapcache_cache_composite_cache_link mapcache_cache_composite_cache_link; +struct mapcache_cache_composite_cache_link { + mapcache_cache *cache; + int minzoom; + int maxzoom; + apr_array_header_t *grids; + apr_table_t *dimensions; /* key/value pairs of dimensions */ +}; + +struct mapcache_cache_composite { + mapcache_cache cache; + apr_array_header_t *cache_links; +}; + +static mapcache_cache_composite_cache_link* _mapcache_cache_link_create(apr_pool_t *pool) { + mapcache_cache_composite_cache_link *cl = apr_pcalloc(pool, sizeof(mapcache_cache_composite_cache_link)); + cl->cache=NULL; + cl->dimensions=NULL; + cl->grids=NULL; + cl->maxzoom=-1; + cl->minzoom=-1; + return cl; +} +/** + * returns the mapcache_cache to use for a given tile + * @param ctx + * @param tile + * @return + */ +static mapcache_cache* _mapcache_composite_cache_get(mapcache_context *ctx, mapcache_cache_composite *cache, mapcache_tile *tile) { + int i; + for(i=0; icache_links->nelts; i++) { + mapcache_cache_composite_cache_link *cache_link = APR_ARRAY_IDX(cache->cache_links,i,mapcache_cache_composite_cache_link*); + if(cache_link->minzoom != -1 && tile->z < cache_link->minzoom) continue; + if(cache_link->maxzoom != -1 && tile->z > cache_link->maxzoom) continue; + if(cache_link->grids) { + int j; + for(j=0;jgrids->nelts;j++) { + char *grid_name = APR_ARRAY_IDX(cache_link->grids,j,char*); + if(!strcmp(tile->grid_link->grid->name,grid_name)) + break; + } + /* not found */ + if(j == cache_link->grids->nelts) continue; + } + if(cache_link->dimensions) { + const apr_array_header_t *array = apr_table_elts(cache_link->dimensions); + apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; + int j; + if(!tile->dimensions) continue; /* the cache link refers to dimensions, but this tile does not have any, it cannot match */ + + for (j = 0; j < array->nelts; j++) { + char *dim = elts[j].key; + char *dimval = elts[j].val; + int k; + for(k=0;kdimensions->nelts;k++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + if(!strcmp(rdim->dimension->name,dim) && !strcmp(rdim->cached_value,dimval)) + break; + } + if(k == tile->dimensions->nelts) break; /* no tile dimension matched the current cache dimension */ + } + if(j != array->nelts) continue; /* we broke out early from the cache dimension loop, so at least one was not correct */ + } + return cache_link->cache; + } + ctx->set_error(ctx, 500, "no cache matches for given tile request"); + return NULL; +} + +static int _mapcache_cache_composite_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + if(GC_HAS_ERROR(ctx) || !subcache) + return MAPCACHE_FAILURE; + return mapcache_cache_tile_exists(ctx, subcache, tile); +} + +static void _mapcache_cache_composite_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + /*delete the tile itself*/ + mapcache_cache_tile_delete(ctx,subcache,tile); +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the composite server + * \private \memberof mapcache_cache_composite + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_composite_tile_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + GC_CHECK_ERROR_RETURN(ctx); + return mapcache_cache_tile_get(ctx,subcache,tile); +} + +static void _mapcache_cache_composite_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + return mapcache_cache_tile_set(ctx,subcache,tile); +} + +static void _mapcache_cache_composite_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) +{ + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + mapcache_cache *subcache; + subcache = _mapcache_composite_cache_get(ctx, cache, &tiles[0]); + GC_CHECK_ERROR(ctx); + return mapcache_cache_tile_multi_set(ctx,subcache,tiles,ntiles); +} + +/** + * \private \memberof mapcache_cache_composite + */ +static void _mapcache_cache_composite_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; + cache->cache_links = apr_array_make(ctx->pool,3,sizeof(mapcache_cache_composite_cache_link*)); + for(cur_node = ezxml_child(node,"cache"); cur_node; cur_node = cur_node->next) { + char *sZoom; + int zoom; + mapcache_cache *refcache = mapcache_configuration_get_cache(config, cur_node->txt); + mapcache_cache_composite_cache_link *cachelink; + if(!refcache) { + ctx->set_error(ctx, 400, "composite cache \"%s\" references cache \"%s\"," + " but it is not configured (hint:referenced caches must be declared before this composite cache in the xml file)", pcache->name, cur_node->txt); + return; + } + cachelink = _mapcache_cache_link_create(ctx->pool); + cachelink->cache = refcache; + + sZoom = (char*)ezxml_attr(cur_node,"max-zoom"); + if(sZoom) { + char *endptr; + zoom = (int)strtol(sZoom,&endptr,10); + if(*endptr != 0 || zoom < 0) { + ctx->set_error(ctx, 400, "failed to parse cache max-zoom %s (expecting a positive integer)", + sZoom); + return; + } + cachelink->maxzoom = zoom; + } + sZoom = (char*)ezxml_attr(cur_node,"min-zoom"); + if(sZoom) { + char *endptr; + zoom = (int)strtol(sZoom,&endptr,10); + if(*endptr != 0 || zoom < 0) { + ctx->set_error(ctx, 400, "failed to parse cache min-zoom %s (expecting a positive integer)", + sZoom); + return; + } + cachelink->minzoom = zoom; + } + sZoom = (char*)ezxml_attr(cur_node,"grids"); + if(sZoom) { + char *grids = apr_pstrdup(ctx->pool,sZoom),*key,*last; + for(key = apr_strtok(grids, ",", &last); key; key = apr_strtok(NULL,",",&last)) { + /*loop through grids*/ + if(!cachelink->grids) { + cachelink->grids =apr_array_make(ctx->pool,1,sizeof(char*)); + } + APR_ARRAY_PUSH(cachelink->grids,char*) = key; + } + } + sZoom = (char*)ezxml_attr(cur_node,"dimensions"); + if(sZoom) { + char *dims = apr_pstrdup(ctx->pool,sZoom),*key,*last; + for(key = apr_strtok(dims, ",", &last); key; key = apr_strtok(NULL,",",&last)) { + char *dimname; + /*loop through dims*/ + if(!cachelink->dimensions) { + cachelink->dimensions =apr_table_make(ctx->pool,1); + } + dimname = key; + while(*key && *key!='=') key++; + if(!(*key)) { + ctx->set_error(ctx,400,"failed to parse composite cache dimensions. expecting dimensions=\"dim1=val1,dim2=val2\""); + return; + } + *key = 0; + key++; + apr_table_set(cachelink->dimensions,dimname,key); + } + } + + APR_ARRAY_PUSH(cache->cache_links,mapcache_cache_composite_cache_link*) = cachelink; + } +} + +/** + * \private \memberof mapcache_cache_composite + */ +static void _mapcache_cache_composite_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ +} + + +/** + * \brief creates and initializes a mapcache_cache_composite + */ +mapcache_cache* mapcache_cache_composite_create(mapcache_context *ctx) +{ + mapcache_cache_composite *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_composite)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate composite cache"); + return NULL; + } + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_COMPOSITE; + cache->cache._tile_delete = _mapcache_cache_composite_tile_delete; + cache->cache._tile_get = _mapcache_cache_composite_tile_get; + cache->cache._tile_exists = _mapcache_cache_composite_tile_exists; + cache->cache._tile_set = _mapcache_cache_composite_tile_set; + cache->cache._tile_multi_set = _mapcache_cache_composite_tile_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_composite_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_composite_configuration_parse_xml; + return (mapcache_cache*)cache; +} diff -Nru mapcache-1.2.1/lib/cache_couchbase.c mapcache-1.6.1/lib/cache_couchbase.c --- mapcache-1.2.1/lib/cache_couchbase.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache_couchbase.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,485 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching support file: couchbase cache backend. + * Author: Michael Downey and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" + +#ifdef USE_COUCHBASE + +#include +#include +#include +#include + +typedef struct mapcache_cache_couchbase mapcache_cache_couchbase; + +/**\class mapcache_cache_couchbase + * \brief a mapcache_cache on couchbase servers + * \implements mapcache_cache + */ +struct mapcache_cache_couchbase { + mapcache_cache cache; +// apr_reslist_t *connection_pool; + char *host; + char *username; + char *password; + char *bucket; + mapcache_context *ctx; +}; + + +typedef struct getStruct +{ + mapcache_buffer *tileBuffer; + libcouchbase_error_t error; +} getStruct_t; + +/* Not sure if we need this. */ +static void _couchbase_error_callback(libcouchbase_t instance, + libcouchbase_error_t error, + const char *errinfo) +{ + /* Ignore timeouts... */ + if (error != LIBCOUCHBASE_ETIMEDOUT) { + fprintf(stderr, "\nFATAL ERROR: %s\n", + libcouchbase_strerror(instance, error)); + if (errinfo && strlen(errinfo) != 0) { + fprintf(stderr, "\t\"%s\"\n", errinfo); + } + } +} + +static void _couchbase_get_callback(libcouchbase_t instance, + const void *cookie, + libcouchbase_error_t error, + const void *key, libcouchbase_size_t nkey, + const void *bytes, libcouchbase_size_t nbytes, + libcouchbase_uint32_t flags, libcouchbase_cas_t cas) +{ + (void)instance; + (void)key; + (void)nkey; + (void)flags; + (void)cas; + + if (cookie) + { + getStruct_t *request = (getStruct_t*)cookie; + + request->error = error; + + if (error == LIBCOUCHBASE_SUCCESS && request->tileBuffer) + { + mapcache_buffer_append(request->tileBuffer, nbytes, (void*)bytes); + } + } +} + +static void _couchbase_store_callback(libcouchbase_t instance, + const void* cookie, + libcouchbase_storage_t unknown, + libcouchbase_error_t error, + const void* unknown2, + libcouchbase_size_t unknown3, + libcouchbase_cas_t cas) +{ + (void)instance; + (void)unknown; + (void)unknown2; + (void)unknown3; + (void)cas; + + libcouchbase_error_t* userError = (libcouchbase_error_t*)cookie; + + *userError = error; +} + +static apr_status_t _couchbase_reslist_get_connection(void **conn_, void *params, apr_pool_t *pool) { + mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)params; + + libcouchbase_t *instance = apr_pcalloc(pool,sizeof(libcouchbase_t)); + const char *host = cache->host; + const char *username = cache->username; + const char *passwd = cache->password; + const char *bucket = "default"; + + *instance = libcouchbase_create(host, username, passwd, bucket, NULL); + if (*instance == NULL) { + return APR_EGENERAL; + } + + libcouchbase_set_error_callback(*instance, _couchbase_error_callback); + libcouchbase_set_get_callback(*instance, _couchbase_get_callback); + libcouchbase_set_storage_callback(*instance, _couchbase_store_callback); + + if (libcouchbase_connect(*instance) != LIBCOUCHBASE_SUCCESS) { + return APR_EGENERAL; + } + + /* Wait for the connect to compelete */ + libcouchbase_wait(*instance); + + *conn_ = instance; + return APR_SUCCESS; +} + +static apr_status_t _couchbase_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) { + libcouchbase_t *instance = (libcouchbase_t*)conn_; + libcouchbase_destroy(*instance); + return APR_SUCCESS; +} + +static libcouchbase_t* _couchbase_get_connection(mapcache_context *ctx, mapcache_tile *tile) +{ + apr_status_t rv; + libcouchbase_t *instance; + mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache; + + rv = apr_reslist_acquire(cache->connection_pool, (void **)&instance); + if(rv != APR_SUCCESS) { + ctx->set_error(ctx, 500, "failed to aquire connection to couchbase backend: %s", ctx->get_error_message(ctx)); + return NULL; + } + + return instance; +} + +static void _couchbase_release_connection(mapcache_tile *tile, libcouchbase_t* instance) +{ + mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache; + apr_reslist_release(cache->connection_pool, (void*)instance); +} + +static void _couchbase_invalidate_connection(mapcache_tile *tile, libcouchbase_t* instance) +{ + mapcache_cache_couchbase* cache = (mapcache_cache_couchbase*)tile->tileset->cache; + apr_reslist_invalidate(cache->connection_pool, (void*)instance); +} + +static int _mapcache_cache_couchbase_has_tile(mapcache_context *ctx, mapcache_tile *tile) { + char *key[1]; + libcouchbase_t *instance; + libcouchbase_error_t error; + size_t keySize[1]; + getStruct_t request; + + key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_FALSE; + } + + keySize[0] = strlen(key[0]); + + instance = _couchbase_get_connection(ctx, tile); + request.tileBuffer = 0; + request.error = LIBCOUCHBASE_KEY_ENOENT; + error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, 0); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error)); + _couchbase_invalidate_connection(tile, instance); + return MAPCACHE_FALSE; + } + + libcouchbase_wait(*instance); + + error = request.error; + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to get key %s: %s", key, libcouchbase_strerror(*instance, error)); + _couchbase_invalidate_connection(tile, instance); + return MAPCACHE_FALSE; + } + + _couchbase_release_connection(tile, instance); + return MAPCACHE_TRUE; +} + +static void _mapcache_cache_couchbase_delete(mapcache_context *ctx, mapcache_tile *tile) { + char *key; + libcouchbase_t *instance; + libcouchbase_error_t error; + + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + GC_CHECK_ERROR(ctx); + + instance = _couchbase_get_connection(ctx, tile); + + error = libcouchbase_remove(*instance, 0, key, strlen(key), 0); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error)); + } + + libcouchbase_wait(*instance); + + error = libcouchbase_get_last_error(*instance); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase: failed to delete key %s: %s", key, libcouchbase_strerror(*instance, error)); + } + + _couchbase_release_connection(tile, instance); +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the couchbase server + * \private \memberof mapcache_cache_couchbase + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_couchbase_get(mapcache_context *ctx, mapcache_tile *tile) { + char *key[1]; + size_t keySize[1]; + libcouchbase_t *instance; + libcouchbase_error_t error; + getStruct_t request; + + key[0] = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + + keySize[0] = strlen(key[0]); + + tile->encoded_data = mapcache_buffer_create(0, ctx->pool); + + libcouchbase_time_t expires = 86400; + if(tile->tileset->auto_expire) + expires = tile->tileset->auto_expire; + + instance = _couchbase_get_connection(ctx, tile); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + + request.tileBuffer = tile->encoded_data; + error = libcouchbase_mget(*instance, &request, 1, (const void * const*)key, keySize, &expires); + if (error != LIBCOUCHBASE_SUCCESS) { + ctx->set_error(ctx, 500, "couchbase cache returned error on mget %s", libcouchbase_strerror(*instance, error)); + _couchbase_invalidate_connection(tile, instance); + return MAPCACHE_FAILURE; + } + + libcouchbase_wait(*instance); + + if(request.error != LIBCOUCHBASE_SUCCESS) { + _couchbase_release_connection(tile, instance); + return MAPCACHE_CACHE_MISS; + } + + if (tile->encoded_data->size == 0) { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "couchbase cache returned 0-length data for tile %d %d %d", tile->x, tile->y, tile->z); + return MAPCACHE_FAILURE; + } + + apr_time_t now = apr_time_now(); + tile->mtime = now; + + _couchbase_release_connection(tile, instance); + return MAPCACHE_SUCCESS; +} + +/** + * \brief push tile data to couchbase + * + * writes the content of mapcache_tile::data to the configured couchbased instance(s) + * \private \memberof mapcache_cache_couchbase + * \sa mapcache_cache::tile_set() + */ +static void _mapcache_cache_couchbase_set(mapcache_context *ctx, mapcache_tile *tile) { + char *key; + libcouchbase_t *instance; + libcouchbase_error_t error; + const int max_retries = 3; + int retries = max_retries; + apr_interval_time_t delay; + + /* set expiration to one day if not configured */ + libcouchbase_time_t expires = 86400; + if(tile->tileset->auto_expire) + expires = tile->tileset->auto_expire; + + mapcache_cache_couchbase *cache = (mapcache_cache_couchbase*)tile->tileset->cache; + key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + GC_CHECK_ERROR(ctx); + + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + + instance = _couchbase_get_connection(ctx, tile); + GC_CHECK_ERROR(ctx); + + do + { + error = libcouchbase_store(*instance, &error, LIBCOUCHBASE_SET, key, strlen(key), tile->encoded_data->buf, tile->encoded_data->size, 0, expires, 0); + if (error != LIBCOUCHBASE_SUCCESS) { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to eror %s.", + tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error)); + return; + } + + libcouchbase_wait(*instance); + + if (error == LIBCOUCHBASE_ETMPFAIL) { + if (retries > 0) { + delay = 100000 * (1 << (max_retries - retries)); // Do an exponential back off of starting at 100 milliseconds + apr_sleep(delay); + } + else { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to %s. Maximum number of retries used.", + tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error)); + return; + } + + --retries; + } + + else if (error != LIBCOUCHBASE_SUCCESS) { + _couchbase_release_connection(tile, instance); + ctx->set_error(ctx, 500, "failed to store tile %d %d %d to couchbase cache %s due to error %s.", + tile->x, tile->y, tile->z, cache->cache.name, libcouchbase_strerror(*instance, error)); + return; + } + } + while (error == LIBCOUCHBASE_ETMPFAIL); + + _couchbase_release_connection(tile, instance); +} + +/** + * \private \memberof mapcache_cache_couchbase + */ +static void _mapcache_cache_couchbase_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { + ezxml_t cur_node; + apr_status_t rv; + mapcache_cache_couchbase *dcache = (mapcache_cache_couchbase*)cache; + int servercount = 0; + + for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { + servercount++; + } + + if(!servercount) { + ctx->set_error(ctx, 400, "couchbase cache %s has no s configured", cache->name); + return; + } + + if(servercount > 1) { + ctx->set_error(ctx, 400, "couchbase cache %s has more than 1 server configured", cache->name); + return; + } + + cur_node = ezxml_child(node, "server"); + ezxml_t xhost = ezxml_child(cur_node, "host"); /* Host should contain server:port */ + ezxml_t xusername = ezxml_child(cur_node, "username"); + ezxml_t xpasswd = ezxml_child(cur_node, "password"); + ezxml_t xbucket = ezxml_child(cur_node, "bucket"); + + if(!xhost || !xhost->txt || ! *xhost->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->host = apr_pstrdup(ctx->pool, xhost->txt); + if (dcache->host == NULL) { + ctx->set_error(ctx, 400, "cache %s: failed to allocate host string!", cache->name); + return; + } + } + + if(xusername && xusername->txt && *xusername->txt) { + dcache->username = apr_pstrdup(ctx->pool, xusername->txt); + } + + if(xpasswd && xpasswd->txt && *xpasswd->txt) { + dcache->password = apr_pstrdup(ctx->pool, xpasswd->txt); + } + + if(xbucket && xbucket->txt && *xbucket->txt) { + dcache->bucket = apr_pstrdup(ctx->pool, xbucket->txt); + } + + dcache->ctx = ctx; + + rv = apr_reslist_create(&(dcache->connection_pool), + 0 /* min */, + 10 /* soft max */, + 200 /* hard max */, + 60*1000000 /*60 seconds, ttl*/, + _couchbase_reslist_get_connection, /* resource constructor */ + _couchbase_reslist_free_connection, /* resource destructor */ + dcache, ctx->pool); + if(rv != APR_SUCCESS) { + ctx->set_error(ctx, 500, "failed to create couchbase connection pool"); + return; + } +} + +/** + * \private \memberof mapcache_cache_couchbase + */ +static void _mapcache_cache_couchbase_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, mapcache_cfg *cfg) { +} + +/** + * \brief creates and initializes a mapcache_couchbase_cache + */ +mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) { + mapcache_cache_couchbase *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_couchbase)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate couchbase cache"); + return NULL; + } + + cache->cache.metadata = apr_table_make(ctx->pool, 3); + cache->cache.type = MAPCACHE_CACHE_COUCHBASE; + cache->cache.tile_get = _mapcache_cache_couchbase_get; + cache->cache.tile_exists = _mapcache_cache_couchbase_has_tile; + cache->cache.tile_set = _mapcache_cache_couchbase_set; + cache->cache.tile_delete = _mapcache_cache_couchbase_delete; + cache->cache.configuration_parse_xml = _mapcache_cache_couchbase_configuration_parse_xml; + cache->cache.configuration_post_config = _mapcache_cache_couchbase_configuration_post_config; + cache->host = NULL; + cache->username = NULL; + cache->password = NULL; + cache->bucket = NULL; + + return (mapcache_cache*)cache; +} + +#else +mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"COUCHBASE support not compiled in this version"); + return NULL; +} +#endif + +/* vim: ai ts=3 sts=3 et sw=3 +*/ diff -Nru mapcache-1.2.1/lib/cache_disk.c mapcache-1.6.1/lib/cache_disk.c --- mapcache-1.2.1/lib/cache_disk.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/cache_disk.c 2017-09-29 21:31:13.000000000 +0000 @@ -39,6 +39,26 @@ #include #endif +/**\class mapcache_cache_disk + * \brief a mapcache_cache on a filesytem + * \implements mapcache_cache + */ +typedef struct mapcache_cache_disk mapcache_cache_disk; +struct mapcache_cache_disk { + mapcache_cache cache; + char *base_directory; + char *filename_template; + int symlink_blank; + int creation_retry; + + /** + * Set filename for a given tile + * \memberof mapcache_cache_disk + */ + void (*tile_key)(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path); +}; + + /** * \brief computes the relative path between two destinations * @@ -111,29 +131,33 @@ * \param path pointer to a char* that will contain the filename * \private \memberof mapcache_cache_disk */ -static void _mapcache_cache_disk_base_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_base_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { *path = apr_pstrcat(ctx->pool, - ((mapcache_cache_disk*)tile->tileset->cache)->base_directory,"/", + cache->base_directory,"/", tile->tileset->name,"/", tile->grid_link->grid->name, NULL); if(tile->dimensions) { - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->val,"/.",'#'); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + char *dimval; + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not set",entry->dimension->name); + return; + } + dimval = mapcache_util_str_sanitize(ctx->pool,entry->cached_value,"/.",'#'); *path = apr_pstrcat(ctx->pool,*path,"/",dimval,NULL); } } } -static void _mapcache_cache_disk_blank_tile_key(mapcache_context *ctx, mapcache_tile *tile, unsigned char *color, char **path) +static void _mapcache_cache_disk_blank_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, unsigned char *color, char **path) { /* not implemented for template caches, as symlink_blank will never be set */ *path = apr_psprintf(ctx->pool,"%s/%s/%s/blanks/%02X%02X%02X%02X.%s", - ((mapcache_cache_disk*)tile->tileset->cache)->base_directory, + cache->base_directory, tile->tileset->name, tile->grid_link->grid->name, color[0], @@ -154,12 +178,11 @@ * \param r * \private \memberof mapcache_cache_disk */ -static void _mapcache_cache_disk_tilecache_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_tilecache_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { - mapcache_cache_disk *dcache = (mapcache_cache_disk*)tile->tileset->cache; - if(dcache->base_directory) { + if(cache->base_directory) { char *start; - _mapcache_cache_disk_base_tile_key(ctx, tile, &start); + _mapcache_cache_disk_base_tile_key(ctx, cache, tile, &start); *path = apr_psprintf(ctx->pool,"%s/%02d/%03d/%03d/%03d/%03d/%03d/%03d.%s", start, tile->z, @@ -171,7 +194,7 @@ tile->y % 1000, tile->tileset->format?tile->tileset->format->extension:"png"); } else { - *path = dcache->filename_template; + *path = cache->filename_template; *path = mapcache_util_str_replace(ctx->pool,*path, "{tileset}", tile->tileset->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{ext}", @@ -197,14 +220,20 @@ *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_z}", apr_psprintf(ctx->pool,"%d", tile->grid_link->grid->nlevels - tile->z - 1)); - if(tile->dimensions) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - char *dimval = apr_pstrdup(ctx->pool,entry->val); - char *iter = dimval; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i, mapcache_requested_dimension*); + char *dimval; + char *single_dim; + char *iter; + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not set",entry->dimension->name); + return; + } + dimval = apr_pstrdup(ctx->pool,entry->cached_value); + iter = dimval; while(*iter) { /* replace dangerous characters by '#' */ if(*iter == '.' || *iter == '/') { @@ -212,7 +241,11 @@ } iter++; } - dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->key,"#",dimval,NULL); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",dimval,NULL); + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(*path,single_dim)) { + *path = mapcache_util_str_replace(ctx->pool,*path, single_dim, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -222,11 +255,10 @@ } } -static void _mapcache_cache_disk_template_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_template_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { - mapcache_cache_disk *dcache = (mapcache_cache_disk*)tile->tileset->cache; - *path = dcache->filename_template; + *path = cache->filename_template; *path = mapcache_util_str_replace(ctx->pool,*path, "{tileset}", tile->tileset->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); *path = mapcache_util_str_replace(ctx->pool,*path, "{ext}", @@ -253,14 +285,20 @@ *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_z}", apr_psprintf(ctx->pool,"%d", tile->grid_link->grid->nlevels - tile->z - 1)); - if(tile->dimensions) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - char *dimval = apr_pstrdup(ctx->pool,entry->val); - char *iter = dimval; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i, mapcache_requested_dimension*); + char *dimval; + char *single_dim; + char *iter; + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not set",entry->dimension->name); + return; + } + dimval = apr_pstrdup(ctx->pool,entry->cached_value); + iter = dimval; while(*iter) { /* replace dangerous characters by '#' */ if(*iter == '.' || *iter == '/') { @@ -268,7 +306,11 @@ } iter++; } - dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->key,"#",dimval,NULL); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",dimval,NULL); + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(*path,single_dim)) { + *path = mapcache_util_str_replace(ctx->pool,*path, single_dim, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -278,12 +320,11 @@ } } -static void _mapcache_cache_disk_arcgis_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_disk_arcgis_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) { - mapcache_cache_disk *dcache = (mapcache_cache_disk*)tile->tileset->cache; - if(dcache->base_directory) { + if(cache->base_directory) { char *start; - _mapcache_cache_disk_base_tile_key(ctx, tile, &start); + _mapcache_cache_disk_base_tile_key(ctx, cache, tile, &start); *path = apr_psprintf(ctx->pool,"%s/L%02d/R%08x/C%08x.%s" , start, tile->z, @@ -297,13 +338,31 @@ } } +static void _mapcache_cache_disk_worldwind_tile_key(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path) +{ + if(cache->base_directory) { + *path = apr_psprintf(ctx->pool,"%s/%d/%04d/%04d_%04d.%s" , + cache->base_directory, + tile->z, + tile->y, + tile->y, + tile->x, + tile->tileset->format?tile->tileset->format->extension:"png"); + } -static int _mapcache_cache_disk_has_tile(mapcache_context *ctx, mapcache_tile *tile) + if(!*path) { + ctx->set_error(ctx,500, "failed to allocate tile key"); + } +} + + +static int _mapcache_cache_disk_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; apr_finfo_t finfo; int rv; - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; + cache->tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } @@ -315,12 +374,13 @@ } } -static void _mapcache_cache_disk_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_disk_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { apr_status_t ret; char errmsg[120]; char *filename; - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; + cache->tile_key(ctx, cache, tile, &filename); GC_CHECK_ERROR(ctx); ret = apr_file_remove(filename,ctx->pool); @@ -337,7 +397,7 @@ * \private \memberof mapcache_cache_disk * \sa mapcache_cache::tile_get() */ -static int _mapcache_cache_disk_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_disk_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; apr_file_t *f; @@ -345,11 +405,13 @@ apr_status_t rv; apr_size_t size; apr_mmap_t *tilemmap; + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + cache->tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FAILURE; } + ctx->log(ctx,MAPCACHE_DEBUG,"checking for tile %s",filename); if((rv=apr_file_open(&f, filename, #ifndef NOMMAP APR_FOPEN_READ, APR_UREAD | APR_GREAD, @@ -359,8 +421,8 @@ ctx->pool)) == APR_SUCCESS) { rv = apr_file_info_get(&finfo, APR_FINFO_SIZE|APR_FINFO_MTIME, f); if(!finfo.size) { - ctx->set_error(ctx, 500, "tile %s has no data",filename); - return MAPCACHE_FAILURE; + ctx->log(ctx, MAPCACHE_WARN, "tile %s has 0 length data",filename); + return MAPCACHE_CACHE_MISS; } size = finfo.size; @@ -373,10 +435,10 @@ * i.e. normally only once. */ tile->mtime = finfo.mtime; - tile->encoded_data = mapcache_buffer_create(size,ctx->pool); #ifndef NOMMAP + tile->encoded_data = mapcache_buffer_create(0,ctx->pool); rv = apr_mmap_create(&tilemmap,f,0,finfo.size,APR_MMAP_READ,ctx->pool); if(rv != APR_SUCCESS) { char errmsg[120]; @@ -386,6 +448,7 @@ tile->encoded_data->buf = tilemmap->mm; tile->encoded_data->size = tile->encoded_data->avail = finfo.size; #else + tile->encoded_data = mapcache_buffer_create(size,ctx->pool); //manually add the data to our buffer apr_file_read(f,(void*)tile->encoded_data->buf,&size); tile->encoded_data->size = size; @@ -418,14 +481,15 @@ * \private \memberof mapcache_cache_disk * \sa mapcache_cache::tile_set() */ -static void _mapcache_cache_disk_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_disk_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { apr_size_t bytes; apr_file_t *f; apr_status_t ret; char errmsg[120]; - char *filename, *hackptr1, *hackptr2=NULL; - const int creation_retry = ((mapcache_cache_disk*)tile->tileset->cache)->creation_retry; + char *filename; + mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; + const int creation_retry = cache->creation_retry; int retry_count_create_file = 0; #ifdef DEBUG @@ -440,29 +504,11 @@ } #endif - ((mapcache_cache_disk*)tile->tileset->cache)->tile_key(ctx, tile, &filename); + cache->tile_key(ctx, cache, tile, &filename); GC_CHECK_ERROR(ctx); - /* find the location of the last '/' in the string */ - hackptr1 = filename; - while(*hackptr1) { - if(*hackptr1 == '/') - hackptr2 = hackptr1; - hackptr1++; - } - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - /* - * apr_dir_make_recursive sometimes sends back this error, although it should not. - * ignore this one - */ - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; - } - } - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); ret = apr_file_remove(filename,ctx->pool); if(ret != APR_SUCCESS && !APR_STATUS_IS_ENOENT(ret)) { @@ -471,34 +517,29 @@ #ifdef HAVE_SYMLINK - if(((mapcache_cache_disk*)tile->tileset->cache)->symlink_blank) { + if(cache->symlink_blank) { if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); } if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { char *blankname; - _mapcache_cache_disk_blank_tile_key(ctx,tile,tile->raw_image->data,&blankname); + int retry_count_create_symlink = 0; + char *blankname_rel = NULL; + _mapcache_cache_disk_blank_tile_key(ctx,cache,tile,tile->raw_image->data,&blankname); if(apr_file_open(&f, blankname, APR_FOPEN_READ, APR_OS_DEFAULT, ctx->pool) != APR_SUCCESS) { + /* create the blank file */ + int isLocked; + void *lock; if(!tile->encoded_data) { tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); GC_CHECK_ERROR(ctx); } - /* create the blank file */ - char *blankdirname = apr_psprintf(ctx->pool, "%s/%s/%s/blanks", - ((mapcache_cache_disk*)tile->tileset->cache)->base_directory, - tile->tileset->name, - tile->grid_link->grid->name); - if(APR_SUCCESS != (ret = apr_dir_make_recursive( - blankdirname, APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create directory %s for blank tiles",blankdirname, apr_strerror(ret,errmsg,120)); - return; - } - } + mapcache_make_parent_dirs(ctx,blankname); + GC_CHECK_ERROR(ctx); /* aquire a lock on the blank file */ - int isLocked = mapcache_lock_or_wait_for_resource(ctx,blankname); + isLocked = mapcache_lock_or_wait_for_resource(ctx,ctx->config->locker,blankname, &lock); if(isLocked == MAPCACHE_TRUE) { @@ -506,7 +547,7 @@ APR_FOPEN_CREATE|APR_FOPEN_WRITE|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY, APR_OS_DEFAULT, ctx->pool)) != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to create file %s: %s",blankname, apr_strerror(ret,errmsg,120)); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); return; /* we could not create the file */ } @@ -514,17 +555,17 @@ ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",blankname, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120)); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); return; /* we could not create the file */ } if(bytes != tile->encoded_data->size) { ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", blankname, (int)bytes, (int)tile->encoded_data->size); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); return; } apr_file_close(f); - mapcache_unlock_resource(ctx,blankname); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); #ifdef DEBUG ctx->log(ctx,MAPCACHE_DEBUG,"created blank tile %s",blankname); #endif @@ -533,12 +574,10 @@ apr_file_close(f); } - int retry_count_create_symlink = 0; /* * compute the relative path between tile and blank tile */ - char *blankname_rel = NULL; blankname_rel = relative_path(ctx,filename, blankname); GC_CHECK_ERROR(ctx); @@ -555,17 +594,8 @@ ctx->set_error(ctx, 500, "failed to link tile %s to %s: %s",filename, blankname_rel, error); return; /* we could not create the file */ } - - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create symlink, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ - } - } - - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); } #ifdef DEBUG ctx->log(ctx, MAPCACHE_DEBUG, "linked blank tile %s to %s",filename,blankname); @@ -582,6 +612,12 @@ GC_CHECK_ERROR(ctx); } + bytes = (apr_size_t)tile->encoded_data->size; + if(bytes == 0) { + ctx->set_error(ctx, 500, "attempting to write 0 length tile to %s",filename); + return; /* we could not create the file */ + } + /* * depending on configuration file creation will retry if it fails. * this can happen on nfs mounted network storage. @@ -597,33 +633,24 @@ ctx->set_error(ctx, 500, "failed to create file %s: %s",filename, apr_strerror(ret,errmsg,120)); return; /* we could not create the file */ } - - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create file, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ - } - } - - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); } - bytes = (apr_size_t)tile->encoded_data->size; ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",filename, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120)); return; /* we could not create the file */ } - if(bytes != tile->encoded_data->size) { - ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", filename, (int)bytes, (int)tile->encoded_data->size); - } ret = apr_file_close(f); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to close file %s:%s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ + } + + if(bytes != tile->encoded_data->size) { + ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", filename, (int)bytes, (int)tile->encoded_data->size); + apr_file_remove(filename, ctx->pool); } } @@ -643,6 +670,8 @@ dcache->tile_key = _mapcache_cache_disk_tilecache_tile_key; } else if(!strcmp(layout,"arcgis")) { dcache->tile_key = _mapcache_cache_disk_arcgis_tile_key; + } else if(!strcmp(layout,"worldwind")) { + dcache->tile_key = _mapcache_cache_disk_worldwind_tile_key; } else if (!strcmp(layout,"template")) { dcache->tile_key = _mapcache_cache_disk_template_tile_key; template_layout = MAPCACHE_TRUE; @@ -706,10 +735,10 @@ cache->creation_retry = 0; cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_DISK; - cache->cache.tile_delete = _mapcache_cache_disk_delete; - cache->cache.tile_get = _mapcache_cache_disk_get; - cache->cache.tile_exists = _mapcache_cache_disk_has_tile; - cache->cache.tile_set = _mapcache_cache_disk_set; + cache->cache._tile_delete = _mapcache_cache_disk_delete; + cache->cache._tile_get = _mapcache_cache_disk_get; + cache->cache._tile_exists = _mapcache_cache_disk_has_tile; + cache->cache._tile_set = _mapcache_cache_disk_set; cache->cache.configuration_post_config = _mapcache_cache_disk_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_disk_configuration_parse_xml; return (mapcache_cache*)cache; diff -Nru mapcache-1.2.1/lib/cache_fallback.c mapcache-1.6.1/lib/cache_fallback.c --- mapcache-1.2.1/lib/cache_fallback.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache_fallback.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,195 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: fallback cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without fallbackriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" + +typedef struct mapcache_cache_fallback mapcache_cache_fallback; + +struct mapcache_cache_fallback { + mapcache_cache cache; + apr_array_header_t *caches; +}; + +static int _mapcache_cache_fallback_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); + return mapcache_cache_tile_exists(ctx, subcache, tile); +} + +static void _mapcache_cache_fallback_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + int i; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + mapcache_cache_tile_delete(ctx, subcache, tile); + ctx->clear_errors(ctx); /* ignore errors */ + } +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the fallback server + * \private \memberof mapcache_cache_fallback + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_fallback_tile_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + mapcache_cache *subcache; + int i,ret; + subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); + ret = mapcache_cache_tile_get(ctx, subcache, tile); + + if(ret == MAPCACHE_FAILURE) { + int first_error = ctx->get_error(ctx); + char *first_error_message = ctx->get_error_message(ctx); + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"GET\" on primary cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Falling back on secondary caches", + APR_ARRAY_IDX(cache->caches,0,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); + ctx->clear_errors(ctx); + for(i=1; icaches->nelts; i++) { + subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if((ret = mapcache_cache_tile_get(ctx, subcache, tile)) == MAPCACHE_FAILURE) { + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"GET\" on fallback cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Continuing with other fallback caches if available", + APR_ARRAY_IDX(cache->caches,0,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); + ctx->clear_errors(ctx); + continue; + } else { + return ret; + } + } + /* all backends failed, return primary error message */ + ctx->set_error(ctx,first_error,first_error_message); + return MAPCACHE_FAILURE; + } else { + /* success or notfound */ + return ret; + } +} + +static void _mapcache_cache_fallback_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + int i,first_error=0; + char *first_error_message; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + mapcache_cache_tile_set(ctx, subcache, tile); + if(GC_HAS_ERROR(ctx)) { + if(!first_error) { + first_error = ctx->get_error(ctx); + first_error_message = ctx->get_error_message(ctx); + } + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"SET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", + APR_ARRAY_IDX(cache->caches,i,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); + ctx->clear_errors(ctx); + } + } + if(first_error) { + ctx->set_error(ctx,first_error,first_error_message); + } +} + +static void _mapcache_cache_fallback_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) +{ + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + int i,first_error=0; + char *first_error_message; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + mapcache_cache_tile_multi_set(ctx, subcache, tiles, ntiles); + if(GC_HAS_ERROR(ctx)) { + if(!first_error) { + first_error = ctx->get_error(ctx); + first_error_message = ctx->get_error_message(ctx); + } + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"MULTISET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", + APR_ARRAY_IDX(cache->caches,i,mapcache_cache*)->name,tiles[0].z,tiles[0].x,tiles[0].y,tiles[0].tileset->name); + ctx->clear_errors(ctx); + } + } + if(first_error) { + ctx->set_error(ctx,first_error,first_error_message); + } +} + +/** + * \private \memberof mapcache_cache_fallback + */ +static void _mapcache_cache_fallback_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; + cache->caches = apr_array_make(ctx->pool,3,sizeof(mapcache_cache*)); + for(cur_node = ezxml_child(node,"cache"); cur_node; cur_node = cur_node->next) { + mapcache_cache *refcache = mapcache_configuration_get_cache(config, cur_node->txt); + if(!refcache) { + ctx->set_error(ctx, 400, "fallback cache \"%s\" references cache \"%s\"," + " but it is not configured (hint:referenced caches must be declared before this fallback cache in the xml file)", pcache->name, cur_node->txt); + return; + } + APR_ARRAY_PUSH(cache->caches,mapcache_cache*) = refcache; + } + if(cache->caches->nelts == 0) { + ctx->set_error(ctx,400,"fallback cache \"%s\" does not reference any child caches", pcache->name); + } +} + +/** + * \private \memberof mapcache_cache_fallback + */ +static void _mapcache_cache_fallback_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ +} + + +/** + * \brief creates and initializes a mapcache_cache_fallback + */ +mapcache_cache* mapcache_cache_fallback_create(mapcache_context *ctx) +{ + mapcache_cache_fallback *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_fallback)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate fallback cache"); + return NULL; + } + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_COMPOSITE; + cache->cache._tile_delete = _mapcache_cache_fallback_tile_delete; + cache->cache._tile_get = _mapcache_cache_fallback_tile_get; + cache->cache._tile_exists = _mapcache_cache_fallback_tile_exists; + cache->cache._tile_set = _mapcache_cache_fallback_tile_set; + cache->cache._tile_multi_set = _mapcache_cache_fallback_tile_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_fallback_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_fallback_configuration_parse_xml; + return (mapcache_cache*)cache; +} + diff -Nru mapcache-1.2.1/lib/cache_memcache.c mapcache-1.6.1/lib/cache_memcache.c --- mapcache-1.2.1/lib/cache_memcache.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/cache_memcache.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,45 +27,140 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_MEMCACHE -#include "mapcache.h" +#include + +typedef struct mapcache_cache_memcache mapcache_cache_memcache; +/**\class mapcache_cache_memcache + * \brief a mapcache_cache on memcached servers + * \implements mapcache_cache + */ -static int _mapcache_cache_memcache_has_tile(mapcache_context *ctx, mapcache_tile *tile) +struct mapcache_cache_memcache_server { + char* host; + int port; +}; + +struct mapcache_cache_memcache { + mapcache_cache cache; + int nservers; + struct mapcache_cache_memcache_server *servers; + int detect_blank; +}; + +struct mapcache_memcache_conn_param { + mapcache_cache_memcache *cache; +}; + +struct mapcache_memcache_pooled_connection { + apr_memcache_t *memcache; + apr_pool_t *pool; +}; + +void mapcache_memcache_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + struct mapcache_memcache_conn_param *param = params; + mapcache_cache_memcache *cache = param->cache; + struct mapcache_memcache_pooled_connection *pc; + int i; + pc = calloc(1,sizeof(struct mapcache_memcache_pooled_connection)); + apr_pool_create(&pc->pool,NULL); + if(APR_SUCCESS != apr_memcache_create(pc->pool, cache->nservers, 0, &(pc->memcache))) { + ctx->set_error(ctx,500,"cache %s: failed to create memcache backend", cache->cache.name); + return; + } + for(i=0; icache->nservers; i++) { + apr_memcache_server_t *server; + if(APR_SUCCESS != apr_memcache_server_create(pc->pool,cache->servers[i].host,cache->servers[i].port,4,5,50,10000,&server)) { + ctx->set_error(ctx,500,"cache %s: failed to create server %s:%d",cache->cache.name,cache->servers[i].host,cache->servers[i].port); + return; + } + if(APR_SUCCESS != apr_memcache_add_server(pc->memcache,server)) { + ctx->set_error(ctx,500,"cache %s: failed to add server %s:%d",cache->cache.name,cache->servers[i].host,cache->servers[i].port); + return; + } + } + *conn_ = pc; +} + +void mapcache_memcache_connection_destructor(void *conn_) { + struct mapcache_memcache_pooled_connection *pc = conn_; + apr_pool_destroy(pc->pool); + free(pc); +} + +static mapcache_pooled_connection* _mapcache_memcache_get_conn(mapcache_context *ctx, + mapcache_cache_memcache *cache, mapcache_tile *tile) { + mapcache_pooled_connection *pc; + struct mapcache_memcache_conn_param param; + + param.cache = cache; + + pc = mapcache_connection_pool_get_connection(ctx,cache->cache.name, mapcache_memcache_connection_constructor, mapcache_memcache_connection_destructor, ¶m); + return pc; +} + +static void _mapcache_memcache_release_conn(mapcache_context *ctx, mapcache_pooled_connection *con) { + mapcache_connection_pool_release_connection(ctx, con); +} + +static int _mapcache_cache_memcache_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *key; char *tmpdata; int rv; size_t tmpdatasize; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); - if(GC_HAS_ERROR(ctx)) { + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); + if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; + mpc = pc->connection; + + key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) { + rv = MAPCACHE_FALSE; + goto cleanup; } - rv = apr_memcache_getp(cache->memcache,ctx->pool,key,&tmpdata,&tmpdatasize,NULL); + rv = apr_memcache_getp(mpc->memcache,ctx->pool,key,&tmpdata,&tmpdatasize,NULL); if(rv != APR_SUCCESS) { - return MAPCACHE_FALSE; + rv = MAPCACHE_FALSE; + goto cleanup; } if(tmpdatasize == 0) { - return MAPCACHE_FALSE; + rv = MAPCACHE_FALSE; + goto cleanup; } - return MAPCACHE_TRUE; + rv = MAPCACHE_TRUE; +cleanup: + _mapcache_memcache_release_conn(ctx,pc); + return rv; } -static void _mapcache_cache_memcache_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_memcache_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *key; int rv; char errmsg[120]; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); GC_CHECK_ERROR(ctx); - rv = apr_memcache_delete(cache->memcache,key,0); + mpc = pc->connection; + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) goto cleanup; + + rv = apr_memcache_delete(mpc->memcache,key,0); if(rv != APR_SUCCESS && rv!= APR_NOTFOUND) { - int code = 500; - ctx->set_error(ctx,code,"memcache: failed to delete key %s: %s", key, apr_strerror(rv,errmsg,120)); + ctx->set_error(ctx,500,"memcache: failed to delete key %s: %s", key, apr_strerror(rv,errmsg,120)); + goto cleanup; } + +cleanup: + _mapcache_memcache_release_conn(ctx,pc); } /** @@ -75,33 +170,55 @@ * \private \memberof mapcache_cache_memcache * \sa mapcache_cache::tile_get() */ -static int _mapcache_cache_memcache_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_memcache_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *key; int rv; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + mapcache_buffer *encoded_data; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FAILURE; } - tile->encoded_data = mapcache_buffer_create(0,ctx->pool); - rv = apr_memcache_getp(cache->memcache,ctx->pool,key,(char**)&tile->encoded_data->buf,&tile->encoded_data->size,NULL); + mpc = pc->connection; + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) { + rv = MAPCACHE_FAILURE; + goto cleanup; + } + encoded_data = mapcache_buffer_create(0,ctx->pool); + rv = apr_memcache_getp(mpc->memcache,ctx->pool,key,(char**)&encoded_data->buf,&encoded_data->size,NULL); if(rv != APR_SUCCESS) { - return MAPCACHE_CACHE_MISS; + rv = MAPCACHE_CACHE_MISS; + goto cleanup; } - if(tile->encoded_data->size == 0) { + if(encoded_data->size == 0) { ctx->set_error(ctx,500,"memcache cache returned 0-length data for tile %d %d %d\n",tile->x,tile->y,tile->z); - return MAPCACHE_FAILURE; + rv = MAPCACHE_FAILURE; + goto cleanup; } /* extract the tile modification time from the end of the data returned */ memcpy( &tile->mtime, - &(((char*)tile->encoded_data->buf)[tile->encoded_data->size-sizeof(apr_time_t)]), + &(((char*)encoded_data->buf)[encoded_data->size-sizeof(apr_time_t)]), sizeof(apr_time_t)); - ((char*)tile->encoded_data->buf)[tile->encoded_data->size+sizeof(apr_time_t)]='\0'; - tile->encoded_data->avail = tile->encoded_data->size; - tile->encoded_data->size -= sizeof(apr_time_t); - return MAPCACHE_SUCCESS; + + ((char*)encoded_data->buf)[encoded_data->size-sizeof(apr_time_t)]='\0'; + encoded_data->avail = encoded_data->size; + encoded_data->size -= sizeof(apr_time_t); + if(((char*)encoded_data->buf)[0] == '#' && encoded_data->size > 1) { + tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy ,encoded_data->buf,&tile->nodata); + } else { + tile->encoded_data = encoded_data; + } + rv = MAPCACHE_SUCCESS; + +cleanup: + _mapcache_memcache_release_conn(ctx,pc); + + return rv; } /** @@ -113,37 +230,63 @@ * \private \memberof mapcache_cache_memcache * \sa mapcache_cache::tile_set() */ -static void _mapcache_cache_memcache_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_memcache_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - char *key; + char *key, *data; int rv; - /* set expiration to one day if not configured */ - int expires = 86400; + /* set no expiration if not configured */ + int expires =0; + apr_time_t now; + mapcache_buffer *encoded_data = NULL; + mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; + mapcache_pooled_connection *pc; + struct mapcache_memcache_pooled_connection *mpc; + pc = _mapcache_memcache_get_conn(ctx,cache,tile); + GC_CHECK_ERROR(ctx); + mpc = pc->connection; + key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); + if(GC_HAS_ERROR(ctx)) goto cleanup; + if(tile->tileset->auto_expire) expires = tile->tileset->auto_expire; - mapcache_cache_memcache *cache = (mapcache_cache_memcache*)tile->tileset->cache; - key = mapcache_util_get_tile_key(ctx, tile,NULL," \r\n\t\f\e\a\b","#"); - GC_CHECK_ERROR(ctx); - if(!tile->encoded_data) { - tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); - GC_CHECK_ERROR(ctx); + if(cache->detect_blank) { + if(!tile->raw_image) { + tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); + GC_CHECK_ERROR(ctx); + } + if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { + encoded_data = mapcache_buffer_create(5,ctx->pool); + ((char*)encoded_data->buf)[0] = '#'; + memcpy(((char*)encoded_data->buf)+1,tile->raw_image->data,4); + encoded_data->size = 5; + } + } + if(!encoded_data) { + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + if(GC_HAS_ERROR(ctx)) goto cleanup; + } + encoded_data = tile->encoded_data; } /* concatenate the current time to the end of the memcache data so we can extract it out * when we re-get the tile */ - char *data = calloc(1,tile->encoded_data->size+sizeof(apr_time_t)); - apr_time_t now = apr_time_now(); + data = calloc(1,encoded_data->size+sizeof(apr_time_t)); + now = apr_time_now(); apr_pool_cleanup_register(ctx->pool, data, (void*)free, apr_pool_cleanup_null); - memcpy(data,tile->encoded_data->buf,tile->encoded_data->size); - memcpy(&(data[tile->encoded_data->size]),&now,sizeof(apr_time_t)); + memcpy(data,encoded_data->buf,encoded_data->size); + memcpy(&(data[encoded_data->size]),&now,sizeof(apr_time_t)); - rv = apr_memcache_set(cache->memcache,key,data,tile->encoded_data->size+sizeof(apr_time_t),expires,0); + rv = apr_memcache_set(mpc->memcache,key,data,encoded_data->size+sizeof(apr_time_t),expires,0); if(rv != APR_SUCCESS) { ctx->set_error(ctx,500,"failed to store tile %d %d %d to memcache cache %s", tile->x,tile->y,tile->z,cache->cache.name); - return; + goto cleanup; } + +cleanup: + _mapcache_memcache_release_conn(ctx,pc); } /** @@ -152,30 +295,25 @@ static void _mapcache_cache_memcache_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { ezxml_t cur_node; + int i = 0; mapcache_cache_memcache *dcache = (mapcache_cache_memcache*)cache; - int servercount = 0; for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { - servercount++; + dcache->nservers++; } - if(!servercount) { + if(!dcache->nservers) { ctx->set_error(ctx,400,"memcache cache %s has no s configured",cache->name); return; } - if(APR_SUCCESS != apr_memcache_create(ctx->pool, servercount, 0, &dcache->memcache)) { - ctx->set_error(ctx,400,"cache %s: failed to create memcache backend", cache->name); - return; - } + dcache->servers = apr_pcalloc(ctx->pool, dcache->nservers * sizeof(struct mapcache_cache_memcache_server)); + for(cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { ezxml_t xhost = ezxml_child(cur_node,"host"); ezxml_t xport = ezxml_child(cur_node,"port"); - const char *host; - apr_memcache_server_t *server; - apr_port_t port; if(!xhost || !xhost->txt || ! *xhost->txt) { ctx->set_error(ctx,400,"cache %s: with no ",cache->name); return; } else { - host = apr_pstrdup(ctx->pool,xhost->txt); + dcache->servers[i].host = apr_pstrdup(ctx->pool,xhost->txt); } if(!xport || !xport->txt || ! *xport->txt) { @@ -188,19 +326,15 @@ ctx->set_error(ctx,400,"failed to parse value %s for memcache cache %s", xport->txt,cache->name); return; } - port = iport; + dcache->servers[i].port = iport; } - if(APR_SUCCESS != apr_memcache_server_create(ctx->pool,host,port,4,5,50,10000,&server)) { - ctx->set_error(ctx,400,"cache %s: failed to create server %s:%d",cache->name,host,port); - return; - } - if(APR_SUCCESS != apr_memcache_add_server(dcache->memcache,server)) { - ctx->set_error(ctx,400,"cache %s: failed to add server %s:%d",cache->name,host,port); - return; - } - if(APR_SUCCESS != apr_memcache_set(dcache->memcache,"mapcache_test_key","mapcache",8,0,0)) { - ctx->set_error(ctx,400,"cache %s: failed to add test key to server %s:%d",cache->name,host,port); - return; + i++; + } + + dcache->detect_blank = 0; + if ((cur_node = ezxml_child(node, "detect_blank")) != NULL) { + if(!strcasecmp(cur_node->txt,"true")) { + dcache->detect_blank = 1; } } } @@ -212,7 +346,7 @@ mapcache_cfg *cfg) { mapcache_cache_memcache *dcache = (mapcache_cache_memcache*)cache; - if(!dcache->memcache || dcache->memcache->ntotal==0) { + if(!dcache->nservers) { ctx->set_error(ctx,400,"cache %s has no servers configured",cache->name); } } @@ -230,15 +364,20 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_MEMCACHE; - cache->cache.tile_get = _mapcache_cache_memcache_get; - cache->cache.tile_exists = _mapcache_cache_memcache_has_tile; - cache->cache.tile_set = _mapcache_cache_memcache_set; - cache->cache.tile_delete = _mapcache_cache_memcache_delete; + cache->cache._tile_get = _mapcache_cache_memcache_get; + cache->cache._tile_exists = _mapcache_cache_memcache_has_tile; + cache->cache._tile_set = _mapcache_cache_memcache_set; + cache->cache._tile_delete = _mapcache_cache_memcache_delete; cache->cache.configuration_post_config = _mapcache_cache_memcache_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_memcache_configuration_parse_xml; return (mapcache_cache*)cache; } +#else +mapcache_cache* mapcache_cache_memcache_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"MEMCACHE support not compiled in this version"); + return NULL; +} #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.2.1/lib/cache_multitier.c mapcache-1.6.1/lib/cache_multitier.c --- mapcache-1.2.1/lib/cache_multitier.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache_multitier.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,165 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: multitier cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without multitierriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" + +typedef struct mapcache_cache_multitier mapcache_cache_multitier; + +struct mapcache_cache_multitier { + mapcache_cache cache; + apr_array_header_t *caches; +}; + + +static int _mapcache_cache_multitier_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + int i; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if(mapcache_cache_tile_exists(ctx, subcache, tile) == MAPCACHE_TRUE) { + return MAPCACHE_TRUE; + } + } + return MAPCACHE_FALSE; +} + +static void _mapcache_cache_multitier_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + int i; + for(i=0; icaches->nelts; i++) { + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + mapcache_cache_tile_delete(ctx, subcache, tile); + ctx->clear_errors(ctx); /* ignore errors */ + } +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the multitier server + * \private \memberof mapcache_cache_multitier + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_multitier_tile_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + mapcache_cache *subcache; + int i,ret; + subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); + ret = mapcache_cache_tile_get(ctx, subcache, tile); + + if(ret == MAPCACHE_CACHE_MISS) { + for(i=1; icaches->nelts; i++) { + subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + if(mapcache_cache_tile_get(ctx, subcache, tile) == MAPCACHE_SUCCESS) { + ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from secondary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + for(--i;i>=0;i--) { + subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); + mapcache_cache_tile_set(ctx, subcache, tile); + ctx->clear_errors(ctx); /* silently ignore these errors */ + ctx->log(ctx,MAPCACHE_DEBUG,"transferring tile (%s,z=%d,y=%d,x=%d) to cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + } + return MAPCACHE_SUCCESS; + } + } + return MAPCACHE_CACHE_MISS; + } else { + //ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from primary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + return ret; + } +} + +static void _mapcache_cache_multitier_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,cache->caches->nelts-1,mapcache_cache*); + return mapcache_cache_tile_set(ctx, subcache, tile); +} + +static void _mapcache_cache_multitier_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) +{ + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,cache->caches->nelts-1,mapcache_cache*); + return mapcache_cache_tile_multi_set(ctx, subcache, tiles, ntiles); +} + +/** + * \private \memberof mapcache_cache_multitier + */ +static void _mapcache_cache_multitier_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; + cache->caches = apr_array_make(ctx->pool,3,sizeof(mapcache_cache*)); + for(cur_node = ezxml_child(node,"cache"); cur_node; cur_node = cur_node->next) { + mapcache_cache *refcache = mapcache_configuration_get_cache(config, cur_node->txt); + if(!refcache) { + ctx->set_error(ctx, 400, "multitier cache \"%s\" references cache \"%s\"," + " but it is not configured (hint:referenced caches must be declared before this multitier cache in the xml file)", pcache->name, cur_node->txt); + return; + } + APR_ARRAY_PUSH(cache->caches,mapcache_cache*) = refcache; + } + if(cache->caches->nelts == 0) { + ctx->set_error(ctx,400,"multitier cache \"%s\" does not reference any child caches", pcache->name); + } +} + +/** + * \private \memberof mapcache_cache_multitier + */ +static void _mapcache_cache_multitier_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ +} + + +/** + * \brief creates and initializes a mapcache_cache_multitier + */ +mapcache_cache* mapcache_cache_multitier_create(mapcache_context *ctx) +{ + mapcache_cache_multitier *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_multitier)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate multitier cache"); + return NULL; + } + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_COMPOSITE; + cache->cache._tile_delete = _mapcache_cache_multitier_tile_delete; + cache->cache._tile_get = _mapcache_cache_multitier_tile_get; + cache->cache._tile_exists = _mapcache_cache_multitier_tile_exists; + cache->cache._tile_set = _mapcache_cache_multitier_tile_set; + cache->cache._tile_multi_set = _mapcache_cache_multitier_tile_multi_set; + cache->cache.configuration_post_config = _mapcache_cache_multitier_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_multitier_configuration_parse_xml; + return (mapcache_cache*)cache; +} + diff -Nru mapcache-1.2.1/lib/cache_rest.c mapcache-1.6.1/lib/cache_rest.c --- mapcache-1.2.1/lib/cache_rest.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache_rest.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,1511 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching: HTTP Rest cache backend. + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 2014 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +typedef struct mapcache_cache_rest mapcache_cache_rest; +typedef struct mapcache_cache_s3 mapcache_cache_s3; +typedef struct mapcache_cache_azure mapcache_cache_azure; +typedef struct mapcache_cache_google mapcache_cache_google; + +typedef enum { + MAPCACHE_REST_METHOD_GET, + MAPCACHE_REST_METHOD_HEAD, + MAPCACHE_REST_METHOD_PUT, + MAPCACHE_REST_METHOD_POST, + MAPCACHE_REST_METHOD_DELETE +} mapcache_rest_method; + +typedef enum { + MAPCACHE_REST_PROVIDER_NONE, + MAPCACHE_REST_PROVIDER_S3, + MAPCACHE_REST_PROVIDER_AZURE, + MAPCACHE_REST_PROVIDER_GOOGLE +} mapcache_rest_provider; + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest); +void hmac_sha256(const unsigned char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + unsigned char *mac, unsigned mac_size); +void hmac_sha1(const char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + void *mac); +void sha_hex_encode(unsigned char *sha, unsigned int sha_size); +char *base64_encode(apr_pool_t *pool, const unsigned char *data, size_t input_length); + +typedef struct mapcache_rest_operation mapcache_rest_operation; +struct mapcache_rest_operation { + apr_table_t *headers; + mapcache_rest_method method; + char *tile_url; + char *header_file; + void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); +}; + +typedef struct mapcache_rest_configuration mapcache_rest_configuration; +struct mapcache_rest_configuration { + apr_table_t *common_headers; + char *tile_url; + char *header_file; + mapcache_rest_operation has_tile; + mapcache_rest_operation get_tile; + mapcache_rest_operation set_tile; + mapcache_rest_operation multi_set_tile; + mapcache_rest_operation delete_tile; + void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); +}; + +/**\class mapcache_cache_rest + * \brief a mapcache_cache on a 3rd party HTTP Rest API + * \implements mapcache_cache + */ +struct mapcache_cache_rest { + mapcache_cache cache; + mapcache_rest_configuration rest; + int use_redirects; + int timeout; + int connection_timeout; + int detect_blank; + mapcache_rest_provider provider; +}; + +struct mapcache_cache_s3 { + mapcache_cache_rest cache; + char *id; + char *secret; + char *region; + char *credentials_file; +}; + +struct mapcache_cache_azure { + mapcache_cache_rest cache; + char *id; + char *secret; + char *container; +}; + +struct mapcache_cache_google { + mapcache_cache_rest cache; + char *access; + char *secret; +}; + +typedef struct { + mapcache_buffer *buffer; + size_t offset; +} buffer_struct; + +struct rest_conn_params { + mapcache_cache_rest *cache; +}; + +void mapcache_rest_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + CURL *curl_handle = curl_easy_init(); + if(!curl_handle) { + ctx->set_error(ctx,500,"failed to create curl handle"); + *conn_ = NULL; + return; + } + *conn_ = curl_handle; +} + +void mapcache_rest_connection_destructor(void *conn_) { + CURL *curl_handle = (CURL*) conn_; + curl_easy_cleanup(curl_handle); +} + +static mapcache_pooled_connection* _rest_get_connection(mapcache_context *ctx, mapcache_cache_rest *cache, mapcache_tile *tile) +{ + mapcache_pooled_connection *pc; + struct rest_conn_params params; + + params.cache = cache; + + pc = mapcache_connection_pool_get_connection(ctx,cache->cache.name,mapcache_rest_connection_constructor, + mapcache_rest_connection_destructor, ¶ms); + if(!GC_HAS_ERROR(ctx) && pc && pc->connection) { + CURL *curl_handle = (CURL*)pc->connection; + curl_easy_reset(curl_handle); + curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, cache->connection_timeout); + curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, cache->timeout); + } + + return pc; +} + +static size_t buffer_read_callback(void *ptr, size_t size, size_t nmemb, void *stream) +{ + buffer_struct *buffer = (buffer_struct*)stream; + void *start = ((char*)(buffer->buffer->buf)) + buffer->offset; + size_t bytes = MAPCACHE_MIN((buffer->buffer->size-buffer->offset),(size * nmemb)); + if(bytes) { + memcpy(ptr,start,bytes); + buffer->offset += bytes; + } + return bytes; +} + +size_t buffer_write_callback(void *ptr, size_t size, size_t nmemb, void *data) +{ + mapcache_buffer *buffer = (mapcache_buffer*)data; + size_t realsize = size * nmemb; + return mapcache_buffer_append(buffer, realsize, ptr); +} + +static void _set_headers(mapcache_context *ctx, CURL *curl, apr_table_t *headers) { + if(!headers) { + return; + } else { + struct curl_slist *curl_headers=NULL; + const apr_array_header_t *array = apr_table_elts(headers); + apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; + int i; + for (i = 0; i < array->nelts; i++) { + if(strlen(elts[i].val) > 0) { + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",elts[i].val,NULL)); + } else { + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,":",NULL)); + } + } + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); + } +} + +static void _put_request(mapcache_context *ctx, CURL *curl, mapcache_buffer *buffer, char *url, apr_table_t *headers) { + CURLcode res; + buffer_struct data; + mapcache_buffer *response; + + data.buffer = buffer; + data.offset = 0; + + response = mapcache_buffer_create(10,ctx->pool); + +#if LIBCURL_VERSION_NUM < 0x071700 + /* + * hack around a bug in curl <= 7.22 where the content-length is added + * a second time even if ti was present in the manually set headers + */ + apr_table_unset(headers, "Content-Length"); +#endif + + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + /* we want to use our own read function */ + curl_easy_setopt(curl, CURLOPT_READFUNCTION, buffer_read_callback); + + /* enable uploading */ + curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); + + /* HTTP PUT please */ + curl_easy_setopt(curl, CURLOPT_PUT, 1L); + + /* don't use an Expect: 100 Continue header */ + apr_table_set(headers, "Expect", ""); + _set_headers(ctx, curl, headers); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + /* now specify which file to upload */ + curl_easy_setopt(curl, CURLOPT_READDATA, &data); + + /* provide the size of the upload, we specicially typecast the value + * to curl_off_t since we must be sure to use the correct data size */ + curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer->size); + + /* send all data to this function */ + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, buffer_write_callback); + + /* we pass our mapcache_buffer struct to the callback function */ + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)response); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest put: %s",curl_easy_strerror(res)); + } else { + long http_code; + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + if(http_code != 200 && http_code != 201 && http_code != 204) { + char *msg = response->buf; + msg[response->size]=0; + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest put with code %ld: %s", http_code, msg); + } + } + +} + +static int _head_request(mapcache_context *ctx, CURL *curl, char *url, apr_table_t *headers) { + + CURLcode res; + long http_code; + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest head %s",curl_easy_strerror(res)); + http_code = 500; + } else { + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + } + + return (int)http_code; +} + +static int _delete_request(mapcache_context *ctx, CURL *curl, char *url, apr_table_t *headers) { + + CURLcode res; + long http_code; + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest delete %s",curl_easy_strerror(res)); + http_code = 500; + } else { + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + } + + return (int)http_code; +} + +static mapcache_buffer* _get_request(mapcache_context *ctx, CURL *curl, char *url, apr_table_t *headers) { + + CURLcode res; + mapcache_buffer *data = NULL; + long http_code; + + _set_headers(ctx, curl, headers); + + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); + + data = mapcache_buffer_create(4000, ctx->pool); + + /* send all data to this function */ + curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, buffer_write_callback); + + /* we pass our mapcache_buffer struct to the callback function */ + curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)data); + + /* specify target URL, and note that this URL should include a file + * name, not only a directory */ + curl_easy_setopt(curl, CURLOPT_URL, url); + + /* Now run off and do what you've been told! */ + res = curl_easy_perform(curl); + /* Check for errors */ + if(res != CURLE_OK) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get: %s",curl_easy_strerror(res)); + data = NULL; + } else { + curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); + /* handle special behavior of s3 */ + if(http_code == 403) { + char *msg = data->buf; + while(msg && *msg) { + if(!strncmp(msg,"NoSuchKey",strlen("NoSuchKey"))) { + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get with code %ld: %s", http_code, msg); + http_code = 404; + data = NULL; + break; + } + msg++; + } + } + if(http_code != 200 && http_code != 404) { + char *msg = data->buf; + msg[data->size]=0; + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get with code %ld: %s", http_code, msg); + } + if(http_code == 404) { + data = NULL; /* not an error */ + } + } + + return data; +} + +/** + * @brief _mapcache_cache_rest_add_headers_from_file populate header table from entries found in file + * @param ctx + * @param file the file from which headers should be read + * @param headers the output table which will be populated + */ +void _mapcache_cache_rest_add_headers_from_file(mapcache_context *ctx, char *file, apr_table_t *headers) { + apr_status_t rv; + apr_file_t *f; + if((rv=apr_file_open(&f, file, APR_FOPEN_READ|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,APR_OS_DEFAULT, + ctx->pool)) == APR_SUCCESS) { + char line[8096]; + while( (rv = apr_file_gets(line,8096,f))== APR_SUCCESS) { + char *header_name=line, *header_val=line, *header_endval; + int found_token = MAPCACHE_FALSE; + /*search for header delimiter (:)*/ + while(header_val && *header_val) { + if(*header_val == ':') { + *header_val = '\0'; + found_token = MAPCACHE_TRUE; + break; + } + header_val++; + } + if(!found_token) { + /* malformed line, silently skip it */ + continue; + } + + header_val++; + + if(!*header_val) { + /* malformed/empty line, skip it */ + continue; + } + + header_endval = header_val; + while(*header_endval) { + if(*header_endval == '\r' || *header_endval == '\n') { + *header_endval = '\0'; + break; + } + header_endval++; + } + + if(!*header_val) { + /* empty header value, skip it */ + continue; + } + + apr_table_set(headers, header_name, header_val); + } + apr_file_close(f); + } else { + ctx->set_error(ctx,500,"rest cache: failed to access header file"); + } +} + +apr_table_t* _mapcache_cache_rest_headers(mapcache_context *ctx, mapcache_tile *tile, mapcache_rest_configuration *config, + mapcache_rest_operation *operation) { + apr_table_t *ret = apr_table_make(ctx->pool,3); + const apr_array_header_t *array; + + if(config->common_headers) { + apr_table_entry_t *elts; + int i; + array = apr_table_elts(config->common_headers); + elts = (apr_table_entry_t *) array->elts; + for (i = 0; i < array->nelts; i++) { + apr_table_set(ret, elts[i].key,elts[i].val); + } + } + if(config->header_file) { + _mapcache_cache_rest_add_headers_from_file(ctx,config->header_file,ret); + if(GC_HAS_ERROR(ctx)) { + return NULL; + } + } + if(operation->headers) { + apr_table_entry_t *elts; + int i; + array = apr_table_elts(operation->headers); + elts = (apr_table_entry_t *) array->elts; + for (i = 0; i < array->nelts; i++) { + apr_table_set(ret, elts[i].key,elts[i].val); + } + } + if(operation->header_file) { + _mapcache_cache_rest_add_headers_from_file(ctx,operation->header_file,ret); + if(GC_HAS_ERROR(ctx)) { + return NULL; + } + } + return ret; +} + +/* Converts an integer value to its hex character*/ +static char to_hex(char code) { + static char hex[] = "0123456789ABCDEF"; + return hex[code & 15]; +} + +/* Returns a url-encoded version of str */ +static char *url_encode(apr_pool_t *pool, char *str) { + char *pstr = str, *buf = apr_pcalloc(pool, strlen(str) * 3 + 1), *pbuf = buf; + while (*pstr) { + if (isalnum(*pstr) || *pstr == '-' || *pstr == '_' || *pstr == '.' || *pstr == '~' || *pstr=='/') + *pbuf++ = *pstr; + else if (*pstr == ' ') + *pbuf++ = '+'; + else + *pbuf++ = '%', *pbuf++ = to_hex(*pstr >> 4), *pbuf++ = to_hex(*pstr & 15); + pstr++; + } + *pbuf = '\0'; + return buf; +} +/** + * \brief return url for given tile given a template + * + * \param tile the tile to get the key from + * \param template the template to build the url from + * \param path pointer to a char* that will contain the url + * \param r + * \private \memberof mapcache_cache_rest + */ +static void _mapcache_cache_rest_tile_url(mapcache_context *ctx, mapcache_tile *tile, mapcache_rest_configuration *config, + mapcache_rest_operation *operation, char **url) +{ + char *slashptr,*path; + int cnt=0; + if(operation && operation->tile_url) { + *url = apr_pstrdup(ctx->pool, operation->tile_url); + } else { + *url = apr_pstrdup(ctx->pool, config->tile_url); + } + + *url = mapcache_util_str_replace(ctx->pool, *url, "{tileset}", tile->tileset->name); + *url = mapcache_util_str_replace(ctx->pool, *url, "{grid}", tile->grid_link->grid->name); + *url = mapcache_util_str_replace(ctx->pool, *url, "{ext}", + tile->tileset->format?tile->tileset->format->extension:"png"); + if(strstr(*url,"{x}")) + *url = mapcache_util_str_replace(ctx->pool,*url, "{x}", + apr_psprintf(ctx->pool,"%d",tile->x)); + else + *url = mapcache_util_str_replace(ctx->pool,*url, "{inv_x}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)); + if(strstr(*url,"{y}")) + *url = mapcache_util_str_replace(ctx->pool,*url, "{y}", + apr_psprintf(ctx->pool,"%d",tile->y)); + else + *url = mapcache_util_str_replace(ctx->pool,*url, "{inv_y}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)); + if(strstr(*url,"{z}")) + *url = mapcache_util_str_replace(ctx->pool,*url, "{z}", + apr_psprintf(ctx->pool,"%d",tile->z)); + else + *url = mapcache_util_str_replace(ctx->pool,*url, "{inv_z}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->nlevels - tile->z - 1)); + if(tile->dimensions) { + int i; + if(strstr(*url,"{dim")) { + char *dimstring=""; + i = tile->dimensions->nelts; + while(i--) { + char *single_dim; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not defined",entry->dimension->name); + return; + } + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",entry->cached_value,NULL); + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(*url,single_dim)) { + *url = mapcache_util_str_replace(ctx->pool,*url, single_dim, entry->cached_value); + } + } + *url = mapcache_util_str_replace(ctx->pool,*url, "{dim}", dimstring); + } + } + /* url-encode everything after the host name */ + + /* find occurence of third "/" in url */ + slashptr = *url; + while(*slashptr) { + if(*slashptr == '/') cnt++; + if(cnt == 3) break; + slashptr++; + } + if(!*slashptr) { + ctx->set_error(ctx,500,"invalid rest url provided, expecting http(s)://server/path format"); + return; + } + path=slashptr; + path = url_encode(ctx->pool,path); + *slashptr=0; + + + *url = apr_pstrcat(ctx->pool,*url,path,NULL); + /*ctx->log(ctx,MAPCACHE_DEBUG,"rest url: %s",*url);*/ +} + + +// Simple comparison function for comparing two HTTP header names that are +// embedded within an HTTP header line, returning true if header1 comes +// before header2 alphabetically, false if not +static int headerle(const char *header1, const char *header2) +{ + while (1) { + if (*header1 == ':') { + return (*header2 == ':'); + } + else if (*header2 == ':') { + return 0; + } + else if (*header2 < *header1) { + return 0; + } + else if (*header2 > *header1) { + return 1; + } + header1++, header2++; + } +} + + +// Replace this with merge sort eventually, it's the best stable sort. But +// since typically the number of elements being sorted is small, it doesn't +// matter that much which sort is used, and gnome sort is the world's simplest +// stable sort. Added a slight twist to the standard gnome_sort - don't go +// forward +1, go forward to the last highest index considered. This saves +// all the string comparisons that would be done "going forward", and thus +// only does the necessary string comparisons to move values back into their +// sorted position. +static void header_gnome_sort(char **headers, int size) +{ + int i = 0, last_highest = 0; + + while (i < size) { + if ((i == 0) || headerle(headers[i - 1], headers[i])) { + i = ++last_highest; + } + else { + char *tmp = headers[i]; + headers[i] = headers[i - 1]; + headers[--i] = tmp; + } + } +} + +static const char* my_apr_table_get(apr_table_t *t, char *key) { + const char *val = apr_table_get(t,key); + if(!val) val = ""; + return val; +} + + +static void _mapcache_cache_google_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) +{ + char *stringToSign, **aheaders, *resource = url, x_amz_date[64]; + const char *head; + const apr_array_header_t *ahead; + apr_table_entry_t *elts; + mapcache_cache_google *google; + time_t now; + struct tm *tnow; + unsigned char sha[65]; + char b64[150]; + int i,nCanonicalHeaders=0,cnt=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_GOOGLE); + google = (mapcache_cache_google*)rcache; + now = time(NULL); + tnow = gmtime(&now); + sha[64]=0; + + strftime(x_amz_date, 64 , "%a, %d %b %Y %H:%M:%S GMT", tnow); + apr_table_set(headers,"x-amz-date",x_amz_date); + + if(!strcmp(method,"PUT")) { + assert(tile->encoded_data); + apr_md5(sha,tile->encoded_data->buf,tile->encoded_data->size); + apr_base64_encode(b64, (char*)sha, 16); + apr_table_set(headers, "Content-MD5", b64); + } + + head = my_apr_table_get(headers, "Content-MD5"); + stringToSign=apr_pstrcat(ctx->pool, method, "\n", head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Type"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + + /* Date: header, left empty as we are using x-amz-date */ + stringToSign=apr_pstrcat(ctx->pool, stringToSign, "\n", NULL); + + ahead = apr_table_elts(headers); + aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); + elts = (apr_table_entry_t *) ahead->elts; + + for (i = 0; i < ahead->nelts; i++) { + if(!strncmp(elts[i].key,"x-amz-",6)) { + char *k = aheaders[nCanonicalHeaders] = apr_pstrdup(ctx->pool, elts[i].key); + while(*k) { + *k = tolower(*k); + k++; + } + nCanonicalHeaders++; + } + } + header_gnome_sort(aheaders, nCanonicalHeaders); + + for(i=0; ipool, stringToSign, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); + } + + /* find occurence of third "/" in url */ + while(*resource) { + if(*resource == '/') cnt++; + if(cnt == 3) break; + resource++; + } + if(!*resource) { + ctx->set_error(ctx,500,"invalid google url provided"); + return; + } + + stringToSign = apr_pstrcat(ctx->pool, stringToSign, resource, NULL); + + hmac_sha1(stringToSign, strlen(stringToSign), (unsigned char*)google->secret, strlen(google->secret), sha); + + apr_base64_encode(b64, (char*)sha, 20); + + + apr_table_set( headers, "Authorization", apr_pstrcat(ctx->pool,"AWS ", google->access, ":", b64, NULL)); +} + +static void _mapcache_cache_azure_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) +{ + char *stringToSign, **aheaders, *canonical_headers="", *canonical_resource=NULL, *resource = url, x_ms_date[64]; + const char *head; + const apr_array_header_t *ahead; + apr_table_entry_t *elts; + mapcache_cache_azure *azure; + time_t now; + struct tm *tnow; + unsigned char sha[65]; + char *b64sign,*keyub64; + int i,nCanonicalHeaders=0,cnt=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_AZURE); + azure = (mapcache_cache_azure*)rcache; + now = time(NULL); + tnow = gmtime(&now); + sha[64]=0; + + strftime(x_ms_date, sizeof(x_ms_date), "%a, %d %b %Y %H:%M:%S GMT", tnow); + apr_table_set(headers,"x-ms-date",x_ms_date); + apr_table_set(headers,"x-ms-version","2009-09-19"); + apr_table_set(headers,"x-ms-blob-type","BlockBlob"); + + stringToSign = apr_pstrcat(ctx->pool, method, "\n", NULL); + head = my_apr_table_get(headers, "Content-Encoding"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Language"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Length"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-MD5"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Type"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Date"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-Modified-Since"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-Match"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-None-Match"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-Unmodified-Since"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Range"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + + ahead = apr_table_elts(headers); + aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); + elts = (apr_table_entry_t *) ahead->elts; + + for (i = 0; i < ahead->nelts; i++) { + char *k; + if(strncmp(elts[i].key,"x-ms-",5) || elts[i].key[5]==0) continue; + k = aheaders[nCanonicalHeaders] = apr_pstrdup(ctx->pool, elts[i].key); + while(*k) { + *k = tolower(*k); + k++; + } + nCanonicalHeaders++; + } + header_gnome_sort(aheaders, nCanonicalHeaders); + + for(i=0; ipool, canonical_headers, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); + } + + /* find occurence of third "/" in url */ + while(*resource) { + if(*resource == '/') cnt++; + if(cnt == 3) break; + resource++; + } + if(!*resource) { + ctx->set_error(ctx,500,"invalid azure url provided"); + return; + } + + canonical_resource = apr_pstrcat(ctx->pool, "/", azure->id, resource, NULL); + + stringToSign = apr_pstrcat(ctx->pool, stringToSign, canonical_headers, canonical_resource, NULL); + + keyub64 = (char*)apr_pcalloc(ctx->pool, apr_base64_decode_len(azure->secret)); + apr_base64_decode(keyub64, azure->secret); + + hmac_sha256((unsigned char*)stringToSign, strlen(stringToSign), (unsigned char*)keyub64, strlen(keyub64), sha, 32); + + b64sign = (char*)apr_pcalloc(ctx->pool, apr_base64_encode_len(32)); + apr_base64_encode(b64sign, (char*)sha, 32); + + + apr_table_set( headers, "Authorization", apr_pstrcat(ctx->pool,"SharedKey ", azure->id, ":", b64sign, NULL)); + + +} + +static void _remove_lineends(char *str) { + if(str) { + size_t len = strlen(str); + while(len>0) { + if(str[len-1] == '\n' || str[len-1] == '\r') { + str[len-1] = '\0'; + len--; + } else { + break; + } + } + } +} + +static void _mapcache_cache_s3_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) +{ + unsigned char sha1[65],sha2[65]; + int cnt=0,i; + time_t now = time(NULL); + struct tm *tnow = gmtime(&now); + const apr_array_header_t *ahead; + char *tosign, *key, *canonical_request, x_amz_date[64], *resource = url, **aheaders, *auth; + apr_table_entry_t *elts; + mapcache_cache_s3 *s3; + char *aws_access_key_id = NULL, *aws_secret_access_key = NULL, *aws_security_token = NULL; + + sha1[64]=sha2[64]=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_S3); + s3 = (mapcache_cache_s3*)rcache; + + if(s3->credentials_file) { + apr_status_t rv; + apr_file_t *f; + if((rv=apr_file_open(&f, s3->credentials_file, + APR_FOPEN_READ|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,APR_OS_DEFAULT, + ctx->pool)) == APR_SUCCESS) { + char line[1024]; + if( (rv = apr_file_gets(line,1024,f))== APR_SUCCESS) { + _remove_lineends(line); + aws_access_key_id = apr_pstrdup(ctx->pool,line); + } + if( (rv = apr_file_gets(line,1024,f))== APR_SUCCESS) { + _remove_lineends(line); + aws_secret_access_key = apr_pstrdup(ctx->pool,line); + } + if( (rv = apr_file_gets(line,1024,f))== APR_SUCCESS) { + _remove_lineends(line); + aws_security_token = apr_pstrdup(ctx->pool,line); + } + apr_file_close(f); + if(!aws_access_key_id || !*aws_access_key_id|| !aws_secret_access_key ||!*aws_secret_access_key) { + ctx->set_error(ctx,500,"failed to read access or secret key from credentials file"); + } + if(aws_security_token && !*aws_security_token) { + aws_security_token = NULL; + } + } else { + ctx->set_error(ctx,500,"failed to access S3 credential config"); + } + } else { + aws_access_key_id = s3->id; + aws_secret_access_key = s3->secret; + aws_security_token = NULL; + } + + if(!strcmp(method,"PUT")) { + assert(tile->encoded_data); + sha256((unsigned char*)tile->encoded_data->buf, tile->encoded_data->size, sha1); + sha_hex_encode(sha1,32); + } else { + /* sha256 hash of empty string */ + memcpy(sha1,"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",64); + } + apr_table_set(headers,"x-amz-content-sha256", (char*)sha1); + /* sha1 contains the hash of the payload */ + + /* find occurence of third "/" in url */ + while(*resource) { + if(*resource == '/') cnt++; + if(cnt == 3) break; + resource++; + } + if(!*resource) { + ctx->set_error(ctx,500,"invalid s3 url provided"); + return; + } + + strftime(x_amz_date, sizeof(x_amz_date), "%Y%m%dT%H%M%SZ", tnow); + apr_table_set(headers, "x-amz-date", x_amz_date); + + if(aws_security_token) { + apr_table_set(headers, "x-amz-security-token", aws_security_token); + } + + canonical_request = apr_pstrcat(ctx->pool, method, "\n" ,resource, "\n\n",NULL); + + ahead = apr_table_elts(headers); + aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); + elts = (apr_table_entry_t *) ahead->elts; + + for (i = 0; i < ahead->nelts; i++) { + char *k = aheaders[i] = apr_pstrdup(ctx->pool, elts[i].key); + while(*k) { + *k = tolower(*k); + k++; + } + } + + header_gnome_sort(aheaders, ahead->nelts); + for(i=0; inelts; i++) { + canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); + } + canonical_request = apr_pstrcat(ctx->pool, canonical_request, "\n", NULL); + for(i=0; inelts; i++) { + if(i==ahead->nelts-1) { + canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],NULL); + } else { + canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],";",NULL); + } + } + canonical_request = apr_pstrcat(ctx->pool, canonical_request, "\n", sha1, NULL); + //printf("canonical request: %s\n",canonical_request); + + tosign = apr_pstrcat(ctx->pool, "AWS4-HMAC-SHA256\n",x_amz_date,"\n",NULL); + x_amz_date[8]=0; + sha256((unsigned char*)canonical_request, strlen(canonical_request), sha1); + sha_hex_encode(sha1,32); + tosign = apr_pstrcat(ctx->pool, tosign, x_amz_date, "/", s3->region, "/s3/aws4_request\n", sha1,NULL); + //printf("key to sign: %s\n",tosign); + + key = apr_pstrcat(ctx->pool, "AWS4", aws_secret_access_key, NULL); + hmac_sha256((unsigned char*)x_amz_date, 8, (unsigned char*)key, strlen(key), sha1, 32); + hmac_sha256((unsigned char*)s3->region, strlen(s3->region), sha1, 32, sha2, 32); + hmac_sha256((unsigned char*)"s3", 2, sha2, 32, sha1, 32); + hmac_sha256((unsigned char*)"aws4_request", 12, sha1, 32, sha2, 32); + hmac_sha256((unsigned char*)tosign, strlen(tosign), sha2, 32, sha1, 32); + sha_hex_encode(sha1,32); + + + auth = apr_pstrcat(ctx->pool, "AWS4-HMAC-SHA256 Credential=",aws_access_key_id,"/",x_amz_date,"/",s3->region,"/s3/aws4_request,SignedHeaders=",NULL); + + for(i=0; inelts; i++) { + if(i==ahead->nelts-1) { + auth = apr_pstrcat(ctx->pool, auth, aheaders[i],NULL); + } else { + auth = apr_pstrcat(ctx->pool, auth, aheaders[i],";",NULL); + } + } + auth = apr_pstrcat(ctx->pool, auth, ",Signature=", sha1, NULL); + + apr_table_set(headers, "Authorization", auth); +} + +static void _mapcache_cache_s3_put_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "PUT", pcache, tile, url, headers); +} +static void _mapcache_cache_s3_get_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "GET", pcache, tile, url, headers); +} +static void _mapcache_cache_s3_head_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "HEAD", pcache, tile, url, headers); +} +static void _mapcache_cache_s3_delete_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_s3_headers_add(ctx, "DELETE", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_put_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "PUT", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_get_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "GET", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_head_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "HEAD", pcache, tile, url, headers); +} +static void _mapcache_cache_azure_delete_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_azure_headers_add(ctx, "DELETE", pcache, tile, url, headers); +} +static void _mapcache_cache_google_put_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "PUT", pcache, tile, url, headers); +} +static void _mapcache_cache_google_get_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "GET", pcache, tile, url, headers); +} +static void _mapcache_cache_google_head_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "HEAD", pcache, tile, url, headers); +} +static void _mapcache_cache_google_delete_headers_add(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers) { + _mapcache_cache_google_headers_add(ctx, "DELETE", pcache, tile, url, headers); +} + + +static int _mapcache_cache_rest_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + int status; + mapcache_pooled_connection *pc; + CURL *curl; + + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.has_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.has_tile); + + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.has_tile.add_headers) { + rcache->rest.has_tile.add_headers(ctx,rcache,tile,url,headers); + } + + pc = _rest_get_connection(ctx, rcache, tile); + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + + curl = pc->connection; + + status = _head_request(ctx, curl, url, headers); + + + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return MAPCACHE_FAILURE; + } + + mapcache_connection_pool_release_connection(ctx,pc); + + if( status == 200) + return MAPCACHE_TRUE; + else + return MAPCACHE_FALSE; +} + +static void _mapcache_cache_rest_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + int status; + mapcache_pooled_connection *pc; + CURL *curl; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.delete_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.delete_tile); + GC_CHECK_ERROR(ctx); + + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.delete_tile.add_headers) { + rcache->rest.delete_tile.add_headers(ctx,rcache,tile,url,headers); + } + + pc = _rest_get_connection(ctx, rcache, tile); + GC_CHECK_ERROR(ctx); + + curl = pc->connection; + + status = _delete_request(ctx, curl, url, headers); + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return; + } + mapcache_connection_pool_release_connection(ctx,pc); + + if(status!=200 && status!=202 && status!=204 && status!=404 && status!=410) { + //ctx->set_error(ctx,500,"rest delete returned code %d", status); + } +} + + +/** + * \brief get file content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored in the file + * \private \memberof mapcache_cache_rest + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_rest_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) +{ + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + mapcache_pooled_connection *pc; + CURL *curl; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.get_tile, &url); + if(tile->allow_redirect && rcache->use_redirects) { + tile->redirect = url; + return MAPCACHE_SUCCESS; + } + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.get_tile); + + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.get_tile.add_headers) { + rcache->rest.get_tile.add_headers(ctx,rcache,tile,url,headers); + } + + pc = _rest_get_connection(ctx, rcache, tile); + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + + curl = pc->connection; + + tile->encoded_data = _get_request(ctx, curl, url, headers); + + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return MAPCACHE_FAILURE; + } + mapcache_connection_pool_release_connection(ctx,pc); + + if(!tile->encoded_data) + return MAPCACHE_CACHE_MISS; + + return MAPCACHE_SUCCESS; +} + +static void _mapcache_cache_rest_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; + char *url; + apr_table_t *headers; + mapcache_pooled_connection *pc; + CURL *curl; + + + if(rcache->detect_blank) { + if(tile->nodata) { + return; + } + if(!tile->raw_image) { + tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); + GC_CHECK_ERROR(ctx); + } + if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { + if(tile->raw_image->data[3] == 0) { + /* We have a blank (uniform) image who's first pixel is fully transparent, thus the whole image is transparent */ + tile->nodata = 1; + return; + } + } + } + + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.set_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.set_tile); + GC_CHECK_ERROR(ctx); + + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + + apr_table_set(headers,"Content-Length",apr_psprintf(ctx->pool,"%lu",tile->encoded_data->size)); + if(tile->tileset->format && tile->tileset->format->mime_type) + apr_table_set(headers, "Content-Type", tile->tileset->format->mime_type); + else { + mapcache_image_format_type imgfmt = mapcache_imageio_header_sniff(ctx,tile->encoded_data); + if(imgfmt == GC_JPEG) { + apr_table_set(headers, "Content-Type", "image/jpeg"); + } else if (imgfmt == GC_PNG) { + apr_table_set(headers, "Content-Type", "image/png"); + } + } + + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.set_tile.add_headers) { + rcache->rest.set_tile.add_headers(ctx,rcache,tile,url,headers); + } + + pc = _rest_get_connection(ctx, rcache, tile); + GC_CHECK_ERROR(ctx); + curl = pc->connection; + + _put_request(ctx, curl, tile->encoded_data, url, headers); + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return; + } + + mapcache_connection_pool_release_connection(ctx,pc); +} + +static void _mapcache_cache_rest_operation_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_rest_operation *op) +{ + ezxml_t cur_node; + if ((cur_node = ezxml_child(node,"headers")) != NULL) { + ezxml_t header_node; + op->headers = apr_table_make(ctx->pool,3); + for(header_node = cur_node->child; header_node; header_node = header_node->sibling) { + apr_table_set(op->headers, header_node->name, header_node->txt); + } + } + if ((cur_node = ezxml_child(node,"header_file")) != NULL) { + op->header_file = apr_pstrdup(ctx->pool, cur_node->txt); + } +} + +/** + * \private \memberof mapcache_cache_rest + */ +static void _mapcache_cache_rest_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_rest *dcache = (mapcache_cache_rest*)cache; + if ((cur_node = ezxml_child(node,"url")) != NULL) { + dcache->rest.tile_url = apr_pstrdup(ctx->pool,cur_node->txt); + } + if ((cur_node = ezxml_child(node,"use_redirects")) != NULL) { + if(!strcasecmp(cur_node->txt,"true")) { + dcache->use_redirects = 1; + } + } + if ((cur_node = ezxml_child(node,"connection_timeout")) != NULL) { + char *endptr; + dcache->connection_timeout = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0 || dcache->connection_timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache \"%s\" (positive integer expected)", + cur_node->txt); + return; + } + } else { + dcache->connection_timeout = 30; + } + + if ((cur_node = ezxml_child(node,"timeout")) != NULL) { + char *endptr; + dcache->timeout = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0 || dcache->timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache \"%s\" (positive integer expected)", + cur_node->txt); + return; + } + } else { + dcache->timeout = 120; + } + + dcache->detect_blank = 0; + if ((cur_node = ezxml_child(node, "detect_blank")) != NULL) { + if(strcasecmp(cur_node->txt,"false")) { + dcache->detect_blank = 1; + } + } + + if ((cur_node = ezxml_child(node,"headers")) != NULL) { + ezxml_t header_node; + dcache->rest.common_headers = apr_table_make(ctx->pool,3); + for(header_node = cur_node->child; header_node; header_node = header_node->sibling) { + apr_table_set(dcache->rest.common_headers, header_node->name, header_node->txt); + } + } + + if ((cur_node = ezxml_child(node,"header_file")) != NULL) { + dcache->rest.header_file = apr_pstrdup(ctx->pool, cur_node->txt); + } + + + for(cur_node = ezxml_child(node,"operation"); cur_node; cur_node = cur_node->next) { + char *type = (char*)ezxml_attr(cur_node,"type"); + if(!type) { + ctx->set_error(ctx,400," with no \"type\" attribute in cache (%s)", cache->name); + return; + } + if(!strcasecmp(type,"put")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.set_tile); + GC_CHECK_ERROR(ctx); + } else if(!strcasecmp(type,"get")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.get_tile); + GC_CHECK_ERROR(ctx); + } else if(!strcasecmp(type,"head")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.has_tile); + GC_CHECK_ERROR(ctx); + } else if(!strcasecmp(type,"delete")) { + _mapcache_cache_rest_operation_parse_xml(ctx,cur_node,cache,&dcache->rest.delete_tile); + GC_CHECK_ERROR(ctx); + } else { + ctx->set_error(ctx,400," with unknown \"type\" (%s) attribute in cache (%s) (expecting put, get, head or delete)", type, cache->name); + return; + } + } + +} + +static void _mapcache_cache_google_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_google *google = (mapcache_cache_google*)cache; + _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); + GC_CHECK_ERROR(ctx); + if ((cur_node = ezxml_child(node,"access")) != NULL) { + google->access = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"google cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + google->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"google cache (%s) is missing required child", cache->name); + return; + } +} + +static void _mapcache_cache_s3_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_s3 *s3 = (mapcache_cache_s3*)cache; + _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); + GC_CHECK_ERROR(ctx); + if ((cur_node = ezxml_child(node,"credentials_file")) != NULL) { + s3->credentials_file = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + if ((cur_node = ezxml_child(node,"id")) != NULL) { + s3->id = apr_pstrdup(ctx->pool, cur_node->txt); + } else if ( getenv("AWS_ACCESS_KEY_ID")) { + s3->id = apr_pstrdup(ctx->pool,getenv("AWS_ACCESS_KEY_ID")); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child or AWS_ACCESS_KEY_ID environment", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + s3->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else if ( getenv("AWS_SECRET_ACCESS_KEY")) { + s3->secret = apr_pstrdup(ctx->pool,getenv("AWS_SECRET_ACCESS_KEY")); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child or AWS_SECRET_ACCESS_KEY environment", cache->name); + return; + } + } + if ((cur_node = ezxml_child(node,"region")) != NULL) { + s3->region = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child", cache->name); + return; + } +} + +static void _mapcache_cache_azure_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_cache_azure *azure = (mapcache_cache_azure*)cache; + _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); + GC_CHECK_ERROR(ctx); + if ((cur_node = ezxml_child(node,"id")) != NULL) { + azure->id = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"azure cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + azure->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"azure cache (%s) is missing required child", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"container")) != NULL) { + azure->container = apr_pstrdup(ctx->pool, cur_node->txt); + } else { + ctx->set_error(ctx,400,"azure cache (%s) is missing required child", cache->name); + return; + } +} + +/** + * \private \memberof mapcache_cache_rest + */ +static void _mapcache_cache_rest_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, + mapcache_cfg *cfg) +{ + mapcache_cache_rest *dcache = (mapcache_cache_rest*)cache; + + + if(!dcache->rest.tile_url) { + if(!dcache->rest.delete_tile.tile_url) { + ctx->set_error(ctx,400, "rest cache (%s) has no global and no for delete_tile operation", cache->name); + return; + } + if(!dcache->rest.get_tile.tile_url) { + ctx->set_error(ctx,400, "rest cache (%s) has no global and no for get_tile operation", cache->name); + return; + } + if(!dcache->rest.set_tile.tile_url) { + ctx->set_error(ctx,400, "rest cache (%s) has no global and no for set_tile operation", cache->name); + return; + } + } +} + +void mapcache_cache_rest_init(mapcache_context *ctx, mapcache_cache_rest *cache) { + cache->use_redirects = 0; + cache->rest.get_tile.method = MAPCACHE_REST_METHOD_GET; + cache->rest.set_tile.method = MAPCACHE_REST_METHOD_PUT; + cache->rest.delete_tile.method = MAPCACHE_REST_METHOD_DELETE; + cache->rest.multi_set_tile.method = MAPCACHE_REST_METHOD_PUT; + cache->rest.has_tile.method = MAPCACHE_REST_METHOD_HEAD; + cache->cache.metadata = apr_table_make(ctx->pool,3); + cache->cache.type = MAPCACHE_CACHE_REST; + cache->cache._tile_delete = _mapcache_cache_rest_delete; + cache->cache._tile_get = _mapcache_cache_rest_get; + cache->cache._tile_exists = _mapcache_cache_rest_has_tile; + cache->cache._tile_set = _mapcache_cache_rest_set; + cache->cache.configuration_post_config = _mapcache_cache_rest_configuration_post_config; + cache->cache.configuration_parse_xml = _mapcache_cache_rest_configuration_parse_xml; +} +/** + * \brief creates and initializes a mapcache_rest_cache + */ +mapcache_cache* mapcache_cache_rest_create(mapcache_context *ctx) +{ + mapcache_cache_rest *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_rest)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate rest cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,cache); + cache->provider = MAPCACHE_REST_PROVIDER_NONE; + return (mapcache_cache*)cache; +} + +/** + * \brief creates and initializes a mapcache_s3_cache + */ +mapcache_cache* mapcache_cache_s3_create(mapcache_context *ctx) +{ + mapcache_cache_s3 *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_s3)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate s3 cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,&cache->cache); + cache->cache.provider = MAPCACHE_REST_PROVIDER_S3; + cache->cache.cache.configuration_parse_xml = _mapcache_cache_s3_configuration_parse_xml; + cache->cache.rest.get_tile.add_headers = _mapcache_cache_s3_get_headers_add; + cache->cache.rest.has_tile.add_headers = _mapcache_cache_s3_head_headers_add; + cache->cache.rest.set_tile.add_headers = _mapcache_cache_s3_put_headers_add; + cache->cache.rest.delete_tile.add_headers = _mapcache_cache_s3_delete_headers_add; + return (mapcache_cache*)cache; +} + +/** + * \brief creates and initializes a mapcache_azure_cache + */ +mapcache_cache* mapcache_cache_azure_create(mapcache_context *ctx) +{ + mapcache_cache_azure *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_azure)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate azure cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,&cache->cache); + cache->cache.provider = MAPCACHE_REST_PROVIDER_AZURE; + cache->cache.cache.configuration_parse_xml = _mapcache_cache_azure_configuration_parse_xml; + cache->cache.rest.get_tile.add_headers = _mapcache_cache_azure_get_headers_add; + cache->cache.rest.has_tile.add_headers = _mapcache_cache_azure_head_headers_add; + cache->cache.rest.set_tile.add_headers = _mapcache_cache_azure_put_headers_add; + cache->cache.rest.delete_tile.add_headers = _mapcache_cache_azure_delete_headers_add; + return (mapcache_cache*)cache; +} + +/** + * \brief creates and initializes a mapcache_google_cache + */ +mapcache_cache* mapcache_cache_google_create(mapcache_context *ctx) +{ + mapcache_cache_google *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_google)); + if(!cache) { + ctx->set_error(ctx, 500, "failed to allocate google cache"); + return NULL; + } + mapcache_cache_rest_init(ctx,&cache->cache); + cache->cache.provider = MAPCACHE_REST_PROVIDER_GOOGLE; + cache->cache.cache.configuration_parse_xml = _mapcache_cache_google_configuration_parse_xml; + cache->cache.rest.get_tile.add_headers = _mapcache_cache_google_get_headers_add; + cache->cache.rest.has_tile.add_headers = _mapcache_cache_google_head_headers_add; + cache->cache.rest.set_tile.add_headers = _mapcache_cache_google_put_headers_add; + cache->cache.rest.delete_tile.add_headers = _mapcache_cache_google_delete_headers_add; + return (mapcache_cache*)cache; +} + + + +/* vim: ts=2 sts=2 et sw=2 +*/ diff -Nru mapcache-1.2.1/lib/cache_riak.c mapcache-1.6.1/lib/cache_riak.c --- mapcache-1.2.1/lib/cache_riak.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache_riak.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,440 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching support file: riak cache backend. + * Author: Michael Downey and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2013 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include "mapcache.h" +#ifdef USE_RIAK + +#include + +#include +#include +#include + +#include +#include + +typedef struct mapcache_cache_riak mapcache_cache_riak; + +/**\class mapcache_cache_riak + * \brief a mapcache_cache for riak servers + * \implements mapcache_cache + */ +struct mapcache_cache_riak { + mapcache_cache cache; + char *host; + int port; + char *key_template; + char *bucket_template; +}; + +struct riak_conn_params { + mapcache_cache_riak *cache; +}; + +void mapcache_riak_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + mapcache_cache_riak *cache = ((struct riak_conn_params*)params)->cache; + struct RIACK_CONNECTION_OPTIONS options; + struct RIACK_CLIENT *client = riack_new_client(0); + + if (client == NULL) { + ctx->set_error(ctx,500,"failed to riack_new_client(0)"); + return; + } + + options.recv_timeout_ms = 2000; + options.send_timeout_ms = 2000; + if (riack_connect(client, cache->host, cache->port, &options) != RIACK_SUCCESS) { + riack_free(client); + ctx->set_error(ctx,500,"failed to riack_connect()"); + return; + } + + if (riack_ping(client) != RIACK_SUCCESS) { + riack_free(client); + ctx->set_error(ctx,500,"failed to riack_ping()"); + return; + } + + *conn_ = client; +} + +void mapcache_riak_connection_destructor(void *conn_) { + struct RIACK_CLIENT *client = (struct RIACK_CLIENT *)conn_; + riack_free(client); +} + +static mapcache_pooled_connection* _riak_get_connection(mapcache_context *ctx, mapcache_cache_riak *cache, mapcache_tile *tile) +{ + mapcache_pooled_connection *pc; + struct riak_conn_params params; + + params.cache = cache; + + pc = mapcache_connection_pool_get_connection(ctx,cache->cache.name,mapcache_riak_connection_constructor, + mapcache_riak_connection_destructor, ¶ms); + + return pc; +} + +static int _mapcache_cache_riak_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + int error; + RIACK_STRING key,bucket; + struct RIACK_GET_OBJECT obj; + struct RIACK_CLIENT *client; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + key.value = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FALSE; + } + key.len = strlen(key.value); + + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + + pc = _riak_get_connection(ctx, cache, tile); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FALSE; + } + client = pc->connection; + + error = riack_get(client, bucket, key, 0, &obj); + + if (error != RIACK_SUCCESS) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + ctx->set_error(ctx, 500, "riak: failed to get key %s: %d", key, error); + return MAPCACHE_FALSE; + } + + if (obj.object.content_count < 1 || obj.object.content[0].data_len == 0) { + error = MAPCACHE_FALSE; + } else { + error = MAPCACHE_TRUE; + } + + riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. + mapcache_connection_pool_release_connection(ctx,pc); + + return error; +} + +static void _mapcache_cache_riak_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + int error; + RIACK_STRING key,bucket; + struct RIACK_CLIENT *client; + struct RIACK_DEL_PROPERTIES properties; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + memset(&properties, 0, sizeof(struct RIACK_DEL_PROPERTIES)); + + + key.value = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); + GC_CHECK_ERROR(ctx); + key.len = strlen(key.value); + + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + + pc = _riak_get_connection(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + client = pc->connection; + + properties.rw_use = 1; + properties.rw = (4294967295 - 3); // Special value meaning "ALL" + error = riack_delete(client, bucket, key, &properties); + + mapcache_connection_pool_release_connection(ctx,pc); + + if (error != RIACK_SUCCESS) { + ctx->set_error(ctx, 500, "riak: failed to delete key %s: %d", key, error); + } +} + +/** + * \brief get content of given tile + * + * fills the mapcache_tile::data of the given tile with content stored on the riak server + * \private \memberof mapcache_cache_riak + * \sa mapcache_cache::tile_get() + */ +static int _mapcache_cache_riak_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + int error; + RIACK_STRING key,bucket; + struct RIACK_GET_OBJECT obj; + struct RIACK_GET_PROPERTIES properties; + struct RIACK_CLIENT *client; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + memset(&properties, 0, sizeof(struct RIACK_GET_PROPERTIES)); + + //Use Buckets defaults instead of setting the read/write attributes + /* + properties.r_use = 1; + properties.r = 1; + */ + + + key.value = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + key.len = strlen(key.value); + + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + + + pc = _riak_get_connection(ctx, cache, tile); + if (GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + client = pc->connection; + + error = riack_get(client, bucket, key, &properties, &obj); + + if (error != RIACK_SUCCESS) + { + mapcache_connection_pool_invalidate_connection(ctx,pc); + ctx->set_error(ctx, 500, "Failed to get tile %s from cache %s due to error %d", key.value, cache->cache.name, error); + return MAPCACHE_FAILURE; + } + + // Check if tile exists. If it doesn't we need to return CACHE_MISS or things go wrong. + // Mapcache doesn't appear to use the has_tile function and uses _get instead so we need + // to do this sort of test here instead of erroring. + if (obj.object.content_count < 1 || obj.object.content[0].data_len == 0) { + riack_free_get_object(client, &obj); // Need to free the object here as well. + mapcache_connection_pool_release_connection(ctx,pc); + return MAPCACHE_CACHE_MISS; + } + + // Copy the data into the buffer + tile->encoded_data = mapcache_buffer_create(0, ctx->pool); + mapcache_buffer_append(tile->encoded_data, obj.object.content[0].data_len, obj.object.content[0].data); + + riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. + + mapcache_connection_pool_release_connection(ctx,pc); + + return MAPCACHE_SUCCESS; +} + +/** + * \brief push tile data to riak + * + * writes the content of mapcache_tile::data to the configured riak instance(s) + * \private \memberof mapcache_cache_riak + * \sa mapcache_cache::tile_set() + */ +static void _mapcache_cache_riak_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { + char *key,*content_type; + int error; + struct RIACK_OBJECT object; + struct RIACK_CONTENT content; + struct RIACK_PUT_PROPERTIES properties; + struct RIACK_CLIENT *client; + RIACK_STRING bucket; + mapcache_pooled_connection *pc; + mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; + + memset(&content, 0, sizeof(struct RIACK_CONTENT)); + memset(&object, 0, sizeof(struct RIACK_OBJECT)); + memset(&properties, 0, sizeof(struct RIACK_PUT_PROPERTIES)); + + //Use Buckets defaults instead of setting the read/write attributes + /* + properties.w_use = 1; + properties.w = 1; + + properties.dw_use = 1; + properties.dw = 0;*/ + + + key = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); + GC_CHECK_ERROR(ctx); + + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + + if (!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + content_type = tile->tileset->format?(tile->tileset->format->mime_type?tile->tileset->format->mime_type:NULL):NULL; + + if(!content_type) { + /* compute the content-type */ + mapcache_image_format_type t = mapcache_imageio_header_sniff(ctx,tile->encoded_data); + if(t == GC_PNG) + content_type = "image/png"; + else if(t == GC_JPEG) + content_type = "image/jpeg"; + } + + pc = _riak_get_connection(ctx, cache, tile); + GC_CHECK_ERROR(ctx); + client = pc->connection; + + // Set up the riak object to put. Need to do this after we get the client connection + object.bucket = bucket; + object.key.value = key; + object.key.len = strlen(key); + object.vclock.len = 0; + object.content_count = 1; + object.content = &content; + content.content_type.value = content_type; + content.content_type.len = content_type?strlen(content_type):0; + content.data = (uint8_t*)tile->encoded_data->buf; + content.data_len = tile->encoded_data->size; + + error = riack_put(client, object, 0, &properties); + + if (error != RIACK_SUCCESS) + mapcache_connection_pool_invalidate_connection(ctx,pc); + else + mapcache_connection_pool_release_connection(ctx,pc); + + if (error != RIACK_SUCCESS) + { + ctx->set_error(ctx, 500, "failed to store tile %s to cache %s due to error %d.", key, cache->cache.name, error); + } +} + +/** + * \private \memberof mapcache_cache_riak + */ +static void _mapcache_cache_riak_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { + ezxml_t cur_node,xhost,xport,xbucket,xkey; + mapcache_cache_riak *dcache = (mapcache_cache_riak*)cache; + int servercount = 0; + + for (cur_node = ezxml_child(node,"server"); cur_node; cur_node = cur_node->next) { + servercount++; + } + + if (!servercount) { + ctx->set_error(ctx, 400, "riak cache %s has no s configured", cache->name); + return; + } + + if (servercount > 1) { + ctx->set_error(ctx, 400, "riak cache %s has more than 1 server configured", cache->name); + return; + } + + cur_node = ezxml_child(node, "server"); + xhost = ezxml_child(cur_node, "host"); /* Host should contain just server */ + xport = ezxml_child(cur_node, "port"); + xbucket = ezxml_child(cur_node, "bucket"); + xkey = ezxml_child(cur_node, "key"); + + if (!xhost || !xhost->txt || ! *xhost->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->host = apr_pstrdup(ctx->pool, xhost->txt); + } + + if (!xport || !xport->txt || ! *xport->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->port = atoi(xport->txt); + } + + if (!xbucket || !xbucket->txt || ! *xbucket->txt) { + ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); + return; + } else { + dcache->bucket_template = apr_pstrdup(ctx->pool, xbucket->txt); + } + + if(xkey && xkey->txt && *xkey->txt) { + dcache->key_template = apr_pstrdup(ctx->pool, xkey->txt); + } +} + +/** + * \private \memberof mapcache_cache_riak + */ +static void _mapcache_cache_riak_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, mapcache_cfg *cfg) { + riack_init(); +} + +/** + * \brief creates and initializes a mapcache_riak_cache + */ +mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx) { + mapcache_cache_riak *cache = apr_pcalloc(ctx->pool,sizeof(mapcache_cache_riak)); + if (!cache) { + ctx->set_error(ctx, 500, "failed to allocate riak cache"); + return NULL; + } + + cache->cache.metadata = apr_table_make(ctx->pool, 3); + cache->cache.type = MAPCACHE_CACHE_RIAK; + cache->cache._tile_get = _mapcache_cache_riak_get; + cache->cache._tile_exists = _mapcache_cache_riak_has_tile; + cache->cache._tile_set = _mapcache_cache_riak_set; + cache->cache._tile_delete = _mapcache_cache_riak_delete; + cache->cache.configuration_parse_xml = _mapcache_cache_riak_configuration_parse_xml; + cache->cache.configuration_post_config = _mapcache_cache_riak_configuration_post_config; + cache->host = NULL; + cache->port = 8087; // Default RIAK port used for protobuf + cache->bucket_template = NULL; + cache->key_template = NULL; + + return (mapcache_cache*)cache; +} + +#else +mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"RIAK support not compiled in this version"); + return NULL; +} +#endif diff -Nru mapcache-1.2.1/lib/cache_sqlite.c mapcache-1.6.1/lib/cache_sqlite.c --- mapcache-1.2.1/lib/cache_sqlite.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/cache_sqlite.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,10 +27,9 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_SQLITE -#include "mapcache.h" #include #include #include @@ -47,17 +46,50 @@ #include -static apr_hash_t *ro_connection_pools = NULL; -static apr_hash_t *rw_connection_pools = NULL; +/**\class mapcache_cache_sqlite + * \brief a mapcache_cache on a filesytem + * \implements mapcache_cache + */ +typedef struct mapcache_cache_sqlite mapcache_cache_sqlite; +typedef struct mapcache_cache_sqlite_stmt mapcache_cache_sqlite_stmt; + +struct mapcache_cache_sqlite_stmt { + char *sql; +}; + +struct sqlite_conn; + +struct mapcache_cache_sqlite { + mapcache_cache cache; + char *dbfile; + mapcache_cache_sqlite_stmt create_stmt; + mapcache_cache_sqlite_stmt exists_stmt; + mapcache_cache_sqlite_stmt get_stmt; + mapcache_cache_sqlite_stmt set_stmt; + mapcache_cache_sqlite_stmt delete_stmt; + apr_table_t *pragmas; + void (*bind_stmt)(mapcache_context *ctx, void *stmt, mapcache_cache_sqlite *cache, mapcache_tile *tile); + int n_prepared_statements; + int detect_blank; + char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; + int count_x, count_y; +}; + + +struct sqlite_conn_params { + mapcache_cache_sqlite *cache; + char *dbfile; + int readonly; +}; struct sqlite_conn { sqlite3 *handle; - int readonly; int nstatements; sqlite3_stmt **prepared_statements; - char *errmsg; }; +#define SQLITE_CONN(pooled_connection) ((struct sqlite_conn*)((pooled_connection)->connection)) + #define HAS_TILE_STMT_IDX 0 #define GET_TILE_STMT_IDX 1 #define SQLITE_SET_TILE_STMT_IDX 2 @@ -70,8 +102,9 @@ #define MBTILES_DEL_TILE_STMT1_IDX 7 #define MBTILES_DEL_TILE_STMT2_IDX 8 +static void _mapcache_cache_sqlite_filename_for_tile(mapcache_context *ctx, mapcache_cache_sqlite *dcache, mapcache_tile *tile, char **path); -static int _sqlite_set_pragmas(apr_pool_t *pool, mapcache_cache_sqlite* cache, struct sqlite_conn *conn) +static void _sqlite_set_pragmas(mapcache_context *ctx, mapcache_cache_sqlite* cache, struct sqlite_conn *conn) { if (cache->pragmas && !apr_is_empty_table(cache->pragmas)) { const apr_array_header_t *elts = apr_table_elts(cache->pragmas); @@ -80,7 +113,7 @@ char *pragma_stmt; for (i = 0; i < elts->nelts; i++) { apr_table_entry_t entry = APR_ARRAY_IDX(elts, i, apr_table_entry_t); - pragma_stmt = apr_psprintf(pool,"PRAGMA %s=%s",entry.key,entry.val); + pragma_stmt = apr_psprintf(ctx->pool,"PRAGMA %s=%s",entry.key,entry.val); do { ret = sqlite3_exec(conn->handle, pragma_stmt, 0, 0, NULL); if (ret != SQLITE_OK && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { @@ -88,78 +121,61 @@ } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret != SQLITE_OK) { - conn->errmsg = apr_psprintf(pool,"failed to execute pragma statement %s",pragma_stmt); - return MAPCACHE_FAILURE; + ctx->set_error(ctx, 500, "failed to execute pragma statement %s",pragma_stmt); + return; } } } - return MAPCACHE_SUCCESS; + return; } -static apr_status_t _sqlite_reslist_get_rw_connection(void **conn_, void *params, apr_pool_t *pool) + + +static void mapcache_sqlite_release_conn(mapcache_context *ctx, mapcache_pooled_connection *conn) { + mapcache_connection_pool_release_connection(ctx,conn); +} + +void mapcache_sqlite_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; int flags; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) params; - struct sqlite_conn *conn = apr_pcalloc(pool, sizeof (struct sqlite_conn)); + struct sqlite_conn_params *sq_params = (struct sqlite_conn_params*)params; + struct sqlite_conn *conn = calloc(1, sizeof (struct sqlite_conn)); *conn_ = conn; - flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_CREATE; - ret = sqlite3_open_v2(cache->dbfile, &conn->handle, flags, NULL); + if(sq_params->readonly) { + flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; + } else { + flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_CREATE; + mapcache_make_parent_dirs(ctx,sq_params->dbfile); + GC_CHECK_ERROR(ctx); + } + ret = sqlite3_open_v2(sq_params->dbfile, &conn->handle, flags, NULL); if (ret != SQLITE_OK) { - conn->errmsg = apr_psprintf(pool,"sqlite backend failed to open db %s: %s", cache->dbfile, sqlite3_errmsg(conn->handle)); - return APR_EGENERAL; + ctx->set_error(ctx,500,"sqlite backend failed to open db %s: %s", sq_params->dbfile, sqlite3_errmsg(conn->handle)); + return; } sqlite3_busy_timeout(conn->handle, 300000); do { - ret = sqlite3_exec(conn->handle, cache->create_stmt.sql, 0, 0, NULL); + ret = sqlite3_exec(conn->handle, sq_params->cache->create_stmt.sql, 0, 0, NULL); if (ret != SQLITE_OK && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { break; } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret != SQLITE_OK) { - conn->errmsg = apr_psprintf(pool, "sqlite backend failed to create db schema on %s: %s", cache->dbfile, sqlite3_errmsg(conn->handle)); - sqlite3_close(conn->handle); - return APR_EGENERAL; - } - conn->readonly = 0; - ret = _sqlite_set_pragmas(pool, cache, conn); - if(ret != MAPCACHE_SUCCESS) { + ctx->set_error(ctx,500, "sqlite backend failed to create db schema on %s: %s", sq_params->dbfile, sqlite3_errmsg(conn->handle)); sqlite3_close(conn->handle); - return APR_EGENERAL; - } - conn->prepared_statements = calloc(cache->n_prepared_statements,sizeof(sqlite3_stmt*)); - conn->nstatements = cache->n_prepared_statements; - - return APR_SUCCESS; -} - -static apr_status_t _sqlite_reslist_get_ro_connection(void **conn_, void *params, apr_pool_t *pool) -{ - int ret; - int flags; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) params; - struct sqlite_conn *conn = apr_pcalloc(pool, sizeof (struct sqlite_conn)); - *conn_ = conn; - flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; - ret = sqlite3_open_v2(cache->dbfile, &conn->handle, flags, NULL); - - if (ret != SQLITE_OK) { - return APR_EGENERAL; + return; } - sqlite3_busy_timeout(conn->handle, 300000); - conn->readonly = 1; - - ret = _sqlite_set_pragmas(pool,cache, conn); - if (ret != MAPCACHE_SUCCESS) { + _sqlite_set_pragmas(ctx, sq_params->cache, conn); + if(GC_HAS_ERROR(ctx)) { sqlite3_close(conn->handle); - return APR_EGENERAL; + return; } - conn->prepared_statements = calloc(cache->n_prepared_statements,sizeof(sqlite3_stmt*)); - conn->nstatements = cache->n_prepared_statements; - return APR_SUCCESS; + conn->prepared_statements = calloc(sq_params->cache->n_prepared_statements,sizeof(sqlite3_stmt*)); + conn->nstatements = sq_params->cache->n_prepared_statements; } -static apr_status_t _sqlite_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_connection_destructor(void *conn_) { struct sqlite_conn *conn = (struct sqlite_conn*) conn_; int i; @@ -170,109 +186,109 @@ } free(conn->prepared_statements); sqlite3_close(conn->handle); - return APR_SUCCESS; } -static struct sqlite_conn* _sqlite_get_conn(mapcache_context *ctx, mapcache_tile* tile, int readonly) { - apr_status_t rv; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = NULL; - apr_reslist_t *pool = NULL; - apr_hash_t *pool_container; - if (readonly) { - pool_container = ro_connection_pools; +static mapcache_pooled_connection* mapcache_sqlite_get_conn(mapcache_context *ctx, mapcache_cache_sqlite *cache, mapcache_tile *tile, int readonly) { + struct sqlite_conn_params params; + mapcache_pooled_connection *pc; + char *key; + _mapcache_cache_sqlite_filename_for_tile(ctx,cache,tile,¶ms.dbfile); + params.cache = cache; + params.readonly = readonly; + if(!strstr(cache->dbfile,"{")) { + key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",cache->cache.name,NULL); } else { - pool_container = rw_connection_pools; + key = apr_pstrcat(ctx->pool,readonly?"ro_":"rw_",params.dbfile,NULL); } - if(!pool_container || NULL == (pool = apr_hash_get(pool_container,cache->cache.name, APR_HASH_KEY_STRING)) ) { -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(!ro_connection_pools) { - ro_connection_pools = apr_hash_make(ctx->process_pool); - rw_connection_pools = apr_hash_make(ctx->process_pool); - } - - /* probably doesn't exist, unless the previous mutex locked us, so we check */ - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - if(!pool) { - /* there where no existing connection pools, create them*/ - rv = apr_reslist_create(&pool, - 0 /* min */, - 10 /* soft max */, - 200 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _sqlite_reslist_get_ro_connection, /* resource constructor */ - _sqlite_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb ro connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(ro_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - rv = apr_reslist_create(&pool, - 0 /* min */, - 1 /* soft max */, - 1 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _sqlite_reslist_get_rw_connection, /* resource constructor */ - _sqlite_reslist_free_connection, /* resource destructor */ - cache, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create bdb rw connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(rw_connection_pools,cache->cache.name,APR_HASH_KEY_STRING,pool); - } -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(readonly) - pool = apr_hash_get(ro_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - else - pool = apr_hash_get(rw_connection_pools,cache->cache.name, APR_HASH_KEY_STRING); - assert(pool); - } - rv = apr_reslist_acquire(pool, (void **) &conn); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to aquire connection to sqlite backend: %s", (conn && conn->errmsg)?conn->errmsg:"unknown error"); - return NULL; - } - return conn; + pc = mapcache_connection_pool_get_connection(ctx,key,mapcache_sqlite_connection_constructor,mapcache_sqlite_connection_destructor,¶ms); + return pc; } -static void _sqlite_release_conn(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn) + +/** + * \brief return sqlite db filename for given tile + * + * \param tile the tile to get the key from + * \param path pointer to a char* that will contain the filename + * \param r + * \private \memberof mapcache_cache_sqlite + */ +static void _mapcache_cache_sqlite_filename_for_tile(mapcache_context *ctx, mapcache_cache_sqlite *dcache, mapcache_tile *tile, char **path) { - apr_reslist_t *pool; - apr_hash_t *pool_container; - if(conn->readonly) { - pool_container = ro_connection_pools; - } else { - pool_container = rw_connection_pools; + *path = dcache->dbfile; + + if(strstr(*path,"{")) { + /* + * generic template substitutions + */ + if(strstr(*path,"{tileset}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{tileset}", + tile->tileset->name); + if(strstr(*path,"{grid}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", + tile->grid_link->grid->name); + if(tile->dimensions && strstr(*path,"{dim")) { + char *dimstring=""; + int i = tile->dimensions->nelts; + while(i--) { + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->cached_value,"/.",'#'); + char *single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",dimval,NULL); + if(strstr(*path,single_dim)) { + *path = mapcache_util_str_replace(ctx->pool,*path, single_dim, dimval); + } + } + *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); + } + + + while(strstr(*path,"{z}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{z}", + apr_psprintf(ctx->pool,dcache->z_fmt,tile->z)); + + if(dcache->count_x > 0) { + while(strstr(*path,"{div_x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{div_x}", + apr_psprintf(ctx->pool,dcache->div_x_fmt,tile->x/dcache->count_x)); + while(strstr(*path,"{inv_div_x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_x}", + apr_psprintf(ctx->pool,dcache->inv_div_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x)); + while(strstr(*path,"{x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{x}", + apr_psprintf(ctx->pool,dcache->x_fmt,tile->x/dcache->count_x*dcache->count_x)); + while(strstr(*path,"{inv_x}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_x}", + apr_psprintf(ctx->pool,dcache->inv_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x*dcache->count_x)); + } + + if(dcache->count_y > 0) { + while(strstr(*path,"{div_y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{div_y}", + apr_psprintf(ctx->pool,dcache->div_y_fmt,tile->y/dcache->count_y)); + while(strstr(*path,"{inv_div_y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_y}", + apr_psprintf(ctx->pool,dcache->inv_div_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y)); + while(strstr(*path,"{y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{y}", + apr_psprintf(ctx->pool,dcache->y_fmt,tile->y/dcache->count_y*dcache->count_y)); + while(strstr(*path,"{inv_y}")) + *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_y}", + apr_psprintf(ctx->pool,dcache->inv_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y*dcache->count_y)); + + } } - pool = apr_hash_get(pool_container,tile->tileset->cache->name, APR_HASH_KEY_STRING); - if (GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(pool, (void*) conn); - } else { - apr_reslist_release(pool, (void*) conn); + if(!*path) { + ctx->set_error(ctx,500, "failed to allocate tile key"); } } + /** * \brief apply appropriate tile properties to the sqlite statement */ -static void _bind_sqlite_params(mapcache_context *ctx, void *vstmt, mapcache_tile *tile) +static void _bind_sqlite_params(mapcache_context *ctx, void *vstmt, mapcache_cache_sqlite *cache, mapcache_tile *tile) { sqlite3_stmt *stmt = vstmt; int paramidx; @@ -311,8 +327,7 @@ paramidx = sqlite3_bind_parameter_index(stmt, ":data"); if (paramidx) { int written = 0; - if(((mapcache_cache_sqlite*)tile->tileset->cache)->detect_blank && tile->grid_link->grid->tile_sx == 256 && - tile->grid_link->grid->tile_sy == 256) { + if(cache->detect_blank) { if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); @@ -339,7 +354,7 @@ } } -static void _bind_mbtiles_params(mapcache_context *ctx, void *vstmt, mapcache_tile *tile) +static void _bind_mbtiles_params(mapcache_context *ctx, void *vstmt, mapcache_cache_sqlite *cache, mapcache_tile *tile) { sqlite3_stmt *stmt = vstmt; int paramidx; @@ -389,26 +404,29 @@ } -static int _mapcache_cache_sqlite_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_sqlite_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 1); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; sqlite3_stmt *stmt; int ret; + pc = mapcache_sqlite_get_conn(ctx,cache,tile,1); if (GC_HAS_ERROR(ctx)) { - if(conn) _sqlite_release_conn(ctx, tile, conn); + if(pc) mapcache_sqlite_release_conn(ctx, pc); if(!tile->tileset->read_only && tile->tileset->source) { /* not an error in this case, as the db file may not have been created yet */ ctx->clear_errors(ctx); } return MAPCACHE_FALSE; } + conn = SQLITE_CONN(pc); stmt = conn->prepared_statements[HAS_TILE_STMT_IDX]; if(!stmt) { sqlite3_prepare(conn->handle, cache->exists_stmt.sql, -1, &conn->prepared_statements[HAS_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[HAS_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW) { ctx->set_error(ctx, 500, "sqlite backend failed on has_tile: %s", sqlite3_errmsg(conn->handle)); @@ -419,46 +437,53 @@ ret = MAPCACHE_TRUE; } sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return ret; } -static void _mapcache_cache_sqlite_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_sqlite_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); - sqlite3_stmt *stmt = conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX]; + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; + sqlite3_stmt *stmt; int ret; + pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); if (GC_HAS_ERROR(ctx)) { - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } + conn = SQLITE_CONN(pc); + stmt = conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX]; if(!stmt) { sqlite3_prepare(conn->handle, cache->delete_stmt.sql, -1, &conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[SQLITE_DEL_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW) { ctx->set_error(ctx, 500, "sqlite backend failed on delete: %s", sqlite3_errmsg(conn->handle)); } sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_mbtiles_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_mbtiles_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; sqlite3_stmt *stmt1,*stmt2,*stmt3; int ret; const char *tile_id; size_t tile_id_size; + pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); if (GC_HAS_ERROR(ctx)) { - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } + conn = SQLITE_CONN(pc); stmt1 = conn->prepared_statements[MBTILES_DEL_TILE_SELECT_STMT_IDX]; stmt2 = conn->prepared_statements[MBTILES_DEL_TILE_STMT1_IDX]; stmt3 = conn->prepared_statements[MBTILES_DEL_TILE_STMT2_IDX]; @@ -474,19 +499,19 @@ /* first extract tile_id from the tile we will delete. We need this because we do not know * if the tile is empty or not. * If it is empty, we will not delete the image blob data from the images table */ - cache->bind_stmt(ctx, stmt1, tile); + cache->bind_stmt(ctx, stmt1, cache, tile); do { ret = sqlite3_step(stmt1); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { ctx->set_error(ctx, 500, "sqlite backend failed on mbtile del 1: %s", sqlite3_errmsg(conn->handle)); sqlite3_reset(stmt1); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret == SQLITE_DONE) { /* tile does not exist, ignore */ sqlite3_reset(stmt1); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } else { tile_id = (const char*) sqlite3_column_text(stmt1, 0); @@ -495,13 +520,13 @@ /* delete the tile from the "map" table */ - cache->bind_stmt(ctx,stmt2, tile); + cache->bind_stmt(ctx,stmt2, cache, tile); ret = sqlite3_step(stmt2); if (ret != SQLITE_DONE && ret != SQLITE_ROW) { ctx->set_error(ctx, 500, "sqlite backend failed on mbtile del 2: %s", sqlite3_errmsg(conn->handle)); sqlite3_reset(stmt1); sqlite3_reset(stmt2); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } @@ -517,7 +542,7 @@ sqlite3_reset(stmt1); sqlite3_reset(stmt2); sqlite3_reset(stmt3); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } } @@ -525,15 +550,14 @@ sqlite3_reset(stmt1); sqlite3_reset(stmt2); sqlite3_reset(stmt3); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _single_mbtile_set(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn) +static void _single_mbtile_set(mapcache_context *ctx, mapcache_cache_sqlite *cache, mapcache_tile *tile, struct sqlite_conn *conn) { sqlite3_stmt *stmt1,*stmt2; - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)tile->tileset->cache; int ret; if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); @@ -552,8 +576,8 @@ stmt1 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT1_IDX]; stmt2 = conn->prepared_statements[MBTILES_SET_EMPTY_TILE_STMT2_IDX]; } - cache->bind_stmt(ctx, stmt1, tile); - cache->bind_stmt(ctx, stmt2, tile); + cache->bind_stmt(ctx, stmt1, cache, tile); + cache->bind_stmt(ctx, stmt2, cache, tile); } else { stmt1 = conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX]; stmt2 = conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX]; @@ -567,8 +591,8 @@ stmt1 = conn->prepared_statements[MBTILES_SET_TILE_STMT1_IDX]; stmt2 = conn->prepared_statements[MBTILES_SET_TILE_STMT2_IDX]; } - cache->bind_stmt(ctx, stmt1, tile); - cache->bind_stmt(ctx, stmt2, tile); + cache->bind_stmt(ctx, stmt1, cache, tile); + cache->bind_stmt(ctx, stmt2, cache, tile); } do { ret = sqlite3_step(stmt1); @@ -596,47 +620,49 @@ sqlite3_reset(stmt2); } -static int _mapcache_cache_sqlite_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_sqlite_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) pcache; struct sqlite_conn *conn; sqlite3_stmt *stmt; int ret; - conn = _sqlite_get_conn(ctx, tile, 1); + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,tile,1); if (GC_HAS_ERROR(ctx)) { - if(conn) _sqlite_release_conn(ctx, tile, conn); if(tile->tileset->read_only || !tile->tileset->source) { + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_FAILURE; } else { /* not an error in this case, as the db file may not have been created yet */ ctx->clear_errors(ctx); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_CACHE_MISS; } } + conn = SQLITE_CONN(pc); stmt = conn->prepared_statements[GET_TILE_STMT_IDX]; if(!stmt) { sqlite3_prepare(conn->handle, cache->get_stmt.sql, -1, &conn->prepared_statements[GET_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[GET_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); do { ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { ctx->set_error(ctx, 500, "sqlite backend failed on get: %s", sqlite3_errmsg(conn->handle)); sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_FAILURE; } } while (ret == SQLITE_BUSY || ret == SQLITE_LOCKED); if (ret == SQLITE_DONE) { sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_CACHE_MISS; } else { const void *blob = sqlite3_column_blob(stmt, 0); int size = sqlite3_column_bytes(stmt, 0); if(size>0 && ((char*)blob)[0] == '#') { - tile->encoded_data = mapcache_empty_png_decode(ctx,blob,&tile->nodata); + tile->encoded_data = mapcache_empty_png_decode(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy ,blob,&tile->nodata); } else { tile->encoded_data = mapcache_buffer_create(size, ctx->pool); memcpy(tile->encoded_data->buf, blob, size); @@ -647,14 +673,13 @@ apr_time_ansi_put(&(tile->mtime), mtime); } sqlite3_reset(stmt); - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return MAPCACHE_SUCCESS; } } -static void _single_sqlitetile_set(mapcache_context *ctx, mapcache_tile *tile, struct sqlite_conn *conn) +static void _single_sqlitetile_set(mapcache_context *ctx, mapcache_cache_sqlite *cache, mapcache_tile *tile, struct sqlite_conn *conn) { - mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*) tile->tileset->cache; sqlite3_stmt *stmt = conn->prepared_statements[SQLITE_SET_TILE_STMT_IDX]; int ret; @@ -662,7 +687,7 @@ sqlite3_prepare(conn->handle, cache->set_stmt.sql, -1, &conn->prepared_statements[SQLITE_SET_TILE_STMT_IDX], NULL); stmt = conn->prepared_statements[SQLITE_SET_TILE_STMT_IDX]; } - cache->bind_stmt(ctx, stmt, tile); + cache->bind_stmt(ctx, stmt, cache, tile); do { ret = sqlite3_step(stmt); if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { @@ -676,29 +701,41 @@ sqlite3_reset(stmt); } -static void _mapcache_cache_sqlite_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_sqlite_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); - GC_CHECK_ERROR(ctx); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + struct sqlite_conn *conn; + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); - _single_sqlitetile_set(ctx,tile,conn); + _single_sqlitetile_set(ctx,cache, tile,conn); if (GC_HAS_ERROR(ctx)) { sqlite3_exec(conn->handle, "ROLLBACK TRANSACTION", 0, 0, 0); } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_sqlite_multi_set(mapcache_context *ctx, mapcache_tile *tiles, int ntiles) +static void _mapcache_cache_sqlite_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { - struct sqlite_conn *conn = _sqlite_get_conn(ctx, &tiles[0], 0); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; int i; - GC_CHECK_ERROR(ctx); + struct sqlite_conn *conn; + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,&tiles[0],0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); for (i = 0; i < ntiles; i++) { mapcache_tile *tile = &tiles[i]; - _single_sqlitetile_set(ctx,tile,conn); + _single_sqlitetile_set(ctx,cache, tile,conn); if(GC_HAS_ERROR(ctx)) break; } if (GC_HAS_ERROR(ctx)) { @@ -706,34 +743,42 @@ } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, &tiles[0], conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_mbtiles_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_mbtiles_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - struct sqlite_conn *conn = _sqlite_get_conn(ctx, tile, 0); - GC_CHECK_ERROR(ctx); + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + struct sqlite_conn *conn; + mapcache_pooled_connection *pc = mapcache_sqlite_get_conn(ctx,cache,tile,0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); if(!tile->raw_image) { tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); if(GC_HAS_ERROR(ctx)) { - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); return; } } sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); - _single_mbtile_set(ctx,tile,conn); + _single_mbtile_set(ctx, cache, tile,conn); if (GC_HAS_ERROR(ctx)) { sqlite3_exec(conn->handle, "ROLLBACK TRANSACTION", 0, 0, 0); } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, tile, conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_mbtiles_multi_set(mapcache_context *ctx, mapcache_tile *tiles, int ntiles) +static void _mapcache_cache_mbtiles_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { - struct sqlite_conn *conn = NULL; int i; + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + mapcache_pooled_connection *pc; + struct sqlite_conn *conn; /* decode/encode image data before going into the sqlite write lock */ for (i = 0; i < ntiles; i++) { @@ -748,12 +793,17 @@ GC_CHECK_ERROR(ctx); } } - conn = _sqlite_get_conn(ctx, &tiles[0], 0); + pc = mapcache_sqlite_get_conn(ctx,cache,&tiles[0],0); + if (GC_HAS_ERROR(ctx)) { + mapcache_sqlite_release_conn(ctx, pc); + return; + } + conn = SQLITE_CONN(pc); sqlite3_exec(conn->handle, "BEGIN TRANSACTION", 0, 0, 0); for (i = 0; i < ntiles; i++) { mapcache_tile *tile = &tiles[i]; - _single_mbtile_set(ctx,tile,conn); + _single_mbtile_set(ctx,cache,tile,conn); if(GC_HAS_ERROR(ctx)) break; } if (GC_HAS_ERROR(ctx)) { @@ -761,32 +811,69 @@ } else { sqlite3_exec(conn->handle, "END TRANSACTION", 0, 0, 0); } - _sqlite_release_conn(ctx, &tiles[0], conn); + mapcache_sqlite_release_conn(ctx, pc); } -static void _mapcache_cache_sqlite_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_sqlite_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) { ezxml_t cur_node; - mapcache_cache_sqlite *dcache; + mapcache_cache_sqlite *cache; sqlite3_initialize(); sqlite3_config(SQLITE_CONFIG_MULTITHREAD); - dcache = (mapcache_cache_sqlite*) cache; + cache = (mapcache_cache_sqlite*) pcache; if ((cur_node = ezxml_child(node, "base")) != NULL) { ctx->set_error(ctx, 500, "sqlite config not supported anymore, use "); return; } if ((cur_node = ezxml_child(node, "dbname_template")) != NULL) { - ctx->set_error(ctx, 500, "sqlite config not supported anymore, use "); + ctx->set_error(ctx, 500, "sqlite config not supported anymore, use a \"multi-sqlite3\" cache type"); return; } if ((cur_node = ezxml_child(node, "dbfile")) != NULL) { - dcache->dbfile = apr_pstrdup(ctx->pool, cur_node->txt); + char *fmt; + cache->dbfile = apr_pstrdup(ctx->pool, cur_node->txt); + fmt = (char*)ezxml_attr(cur_node,"x_fmt"); + if(fmt && *fmt) { + cache->x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"y_fmt"); + if(fmt && *fmt) { + cache->y_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"z_fmt"); + if(fmt && *fmt) { + cache->z_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_x_fmt"); + if(fmt && *fmt) { + cache->inv_x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_y_fmt"); + if(fmt && *fmt) { + cache->inv_y_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"div_x_fmt"); + if(fmt && *fmt) { + cache->div_x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"div_y_fmt"); + if(fmt && *fmt) { + cache->div_y_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_div_x_fmt"); + if(fmt && *fmt) { + cache->inv_div_x_fmt = apr_pstrdup(ctx->pool,fmt); + } + fmt = (char*)ezxml_attr(cur_node,"inv_div_y_fmt"); + if(fmt && *fmt) { + cache->inv_div_y_fmt = apr_pstrdup(ctx->pool,fmt); + } } - dcache->detect_blank = 0; + cache->detect_blank = 0; if ((cur_node = ezxml_child(node, "detect_blank")) != NULL) { - if(!strcasecmp(cur_node->txt,"true")) { - dcache->detect_blank = 1; + if(strcasecmp(cur_node->txt,"false")) { + cache->detect_blank = 1; } } @@ -796,20 +883,53 @@ } } if ((cur_node = ezxml_child(node, "pragma")) != NULL) { - dcache->pragmas = apr_table_make(ctx->pool,1); + cache->pragmas = apr_table_make(ctx->pool,1); while(cur_node) { char *name = (char*)ezxml_attr(cur_node,"name"); if(!name || !cur_node->txt || !strlen(cur_node->txt)) { ctx->set_error(ctx,500," missing name attribute"); return; } - apr_table_set(dcache->pragmas,name,cur_node->txt); + apr_table_set(cache->pragmas,name,cur_node->txt); cur_node = cur_node->next; } } - if (!dcache->dbfile) { - ctx->set_error(ctx, 500, "sqlite cache \"%s\" is missing entry", cache->name); - return; + if ((cur_node = ezxml_child(node, "queries")) != NULL) { + ezxml_t query_node; + if ((query_node = ezxml_child(cur_node, "exists")) != NULL) { + cache->exists_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "get")) != NULL) { + cache->get_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "set")) != NULL) { + cache->set_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "delete")) != NULL) { + cache->delete_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + if ((query_node = ezxml_child(cur_node, "create")) != NULL) { + cache->create_stmt.sql = apr_pstrdup(ctx->pool,query_node->txt); + } + } + + cur_node = ezxml_child(node,"xcount"); + if(cur_node && cur_node->txt && *cur_node->txt) { + char *endptr; + cache->count_x = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0) { + ctx->set_error(ctx,400,"failed to parse xcount value %s for sqlite cache %s", cur_node->txt,cache->cache.name); + return; + } + } + cur_node = ezxml_child(node,"ycount"); + if(cur_node && cur_node->txt && *cur_node->txt) { + char *endptr; + cache->count_y = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0) { + ctx->set_error(ctx,400,"failed to parse ycount value %s for sqlite cache %s", cur_node->txt,cache->cache.name); + return; + } } } @@ -817,18 +937,24 @@ * \private \memberof mapcache_cache_sqlite */ static void _mapcache_cache_sqlite_configuration_post_config(mapcache_context *ctx, - mapcache_cache *cache, mapcache_cfg *cfg) + mapcache_cache *pcache, mapcache_cfg *cfg) { + mapcache_cache_sqlite *cache = (mapcache_cache_sqlite*)pcache; + if (!cache->dbfile) { + ctx->set_error(ctx, 500, "sqlite cache \"%s\" is missing entry", pcache->name); + return; + } } /** * \private \memberof mapcache_cache_sqlite */ static void _mapcache_cache_mbtiles_configuration_post_config(mapcache_context *ctx, - mapcache_cache *cache, mapcache_cfg *cfg) + mapcache_cache *pcache, mapcache_cfg *cfg) { /* check that only one tileset/grid references this cache, as mbtiles does not support multiple tilesets/grids per cache */ +#ifdef FIXME int nrefs = 0; apr_hash_index_t *tileseti = apr_hash_first(ctx->pool,cfg->tilesets); while(tileseti) { @@ -849,12 +975,9 @@ } tileseti = apr_hash_next(tileseti); } - +#endif } -/** - * \brief creates and initializes a mapcache_sqlite_cache - */ mapcache_cache* mapcache_cache_sqlite_create(mapcache_context *ctx) { mapcache_cache_sqlite *cache = apr_pcalloc(ctx->pool, sizeof (mapcache_cache_sqlite)); @@ -864,11 +987,11 @@ } cache->cache.metadata = apr_table_make(ctx->pool, 3); cache->cache.type = MAPCACHE_CACHE_SQLITE; - cache->cache.tile_delete = _mapcache_cache_sqlite_delete; - cache->cache.tile_get = _mapcache_cache_sqlite_get; - cache->cache.tile_exists = _mapcache_cache_sqlite_has_tile; - cache->cache.tile_set = _mapcache_cache_sqlite_set; - cache->cache.tile_multi_set = _mapcache_cache_sqlite_multi_set; + cache->cache._tile_delete = _mapcache_cache_sqlite_delete; + cache->cache._tile_get = _mapcache_cache_sqlite_get; + cache->cache._tile_exists = _mapcache_cache_sqlite_has_tile; + cache->cache._tile_set = _mapcache_cache_sqlite_set; + cache->cache._tile_multi_set = _mapcache_cache_sqlite_multi_set; cache->cache.configuration_post_config = _mapcache_cache_sqlite_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_sqlite_configuration_parse_xml; cache->create_stmt.sql = apr_pstrdup(ctx->pool, @@ -884,9 +1007,15 @@ cache->n_prepared_statements = 4; cache->bind_stmt = _bind_sqlite_params; cache->detect_blank = 1; - return (mapcache_cache*) cache; + cache->x_fmt = cache->y_fmt = cache->z_fmt + = cache->inv_x_fmt = cache->inv_y_fmt + = cache->div_x_fmt = cache->div_y_fmt + = cache->inv_div_x_fmt = cache->inv_div_y_fmt = apr_pstrdup(ctx->pool,"%d"); + cache->count_x = cache->count_y = -1; + return (mapcache_cache*)cache; } + /** * \brief creates and initializes a mapcache_sqlite_cache */ @@ -897,9 +1026,9 @@ return NULL; } cache->cache.configuration_post_config = _mapcache_cache_mbtiles_configuration_post_config; - cache->cache.tile_set = _mapcache_cache_mbtiles_set; - cache->cache.tile_multi_set = _mapcache_cache_mbtiles_multi_set; - cache->cache.tile_delete = _mapcache_cache_mbtiles_delete; + cache->cache._tile_set = _mapcache_cache_mbtiles_set; + cache->cache._tile_multi_set = _mapcache_cache_mbtiles_multi_set; + cache->cache._tile_delete = _mapcache_cache_mbtiles_delete; cache->create_stmt.sql = apr_pstrdup(ctx->pool, "create table if not exists images(tile_id text, tile_data blob, primary key(tile_id));"\ "CREATE TABLE IF NOT EXISTS map (zoom_level integer, tile_column integer, tile_row integer, tile_id text, foreign key(tile_id) references images(tile_id), primary key(tile_row,tile_column,zoom_level));"\ @@ -916,6 +1045,17 @@ cache->bind_stmt = _bind_mbtiles_params; return (mapcache_cache*) cache; } +#else + +mapcache_cache* mapcache_cache_sqlite_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"SQLITE support not compiled in this version"); + return NULL; +} +mapcache_cache* mapcache_cache_mbtiles_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"SQLITE (MBtiles) support not compiled in this version"); + return NULL; +} + #endif diff -Nru mapcache-1.2.1/lib/cache_tiff.c mapcache-1.6.1/lib/cache_tiff.c --- mapcache-1.2.1/lib/cache_tiff.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/cache_tiff.c 2017-09-29 21:31:13.000000000 +0000 @@ -3,10 +3,13 @@ * * Project: MapServer * Purpose: MapCache tile caching: tiled tiff filesytem cache backend. - * Author: Thomas Bonfort, Frank Warmerdam and the MapServer team. + * Author: Thomas Bonfort + * Frank Warmerdam + * Even Rouault + * MapServer team. * ****************************************************************************** - * Copyright (c) 2011 Regents of the University of Minnesota. + * Copyright (c) 2011-2017 Regents of the University of Minnesota. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -27,10 +30,9 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_TIFF -#include "mapcache.h" #include #include #include @@ -39,6 +41,12 @@ #include #include +#ifdef USE_GDAL +#include "cpl_vsi.h" +#include "cpl_conv.h" +#define CPL_SERV_H_INCLUDED +#endif + #ifdef USE_GEOTIFF #include "xtiffio.h" #include "geovalues.h" @@ -53,6 +61,278 @@ #define MyTIFFClose TIFFClose #endif +typedef enum +{ + MAPCACHE_TIFF_STORAGE_FILE, + MAPCACHE_TIFF_STORAGE_REST, + MAPCACHE_TIFF_STORAGE_GOOGLE +} mapcache_cache_tiff_storage_type; + + +typedef struct mapcache_cache_tiff mapcache_cache_tiff; + +struct mapcache_cache_tiff { + mapcache_cache cache; + char *filename_template; + char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; + int count_x; + int count_y; + mapcache_image_format_jpeg *format; + mapcache_locker *locker; + struct { + mapcache_cache_tiff_storage_type type; + int connection_timeout; + int timeout; + char *header_file; + union + { + struct + { + char* access; + char* secret; + } google; + } u; + } storage; +}; + +#ifdef USE_GDAL + +static tsize_t +_tiffReadProc( thandle_t th, tdata_t buf, tsize_t size ) +{ + VSILFILE* fp = (VSILFILE*)th; + return VSIFReadL( buf, 1, size, fp ); +} + +static tsize_t +_tiffWriteProc( thandle_t th, tdata_t buf, tsize_t size ) +{ + VSILFILE* fp = (VSILFILE*)th; + return VSIFWriteL( buf, 1, size, fp ); +} + +static toff_t +_tiffSeekProc( thandle_t th, toff_t off, int whence ) +{ + VSILFILE* fp = (VSILFILE*)th; + if( VSIFSeekL( fp, off, whence ) != 0 ) + { + TIFFErrorExt( th, "_tiffSeekProc", "%s", VSIStrerror( errno ) ); + return (toff_t)( -1 ); + } + return VSIFTellL( fp ); +} + +static int +_tiffCloseProc( thandle_t th ) +{ + VSILFILE* fp = (VSILFILE*)th; + VSIFCloseL(fp); + return 0; +} + +static toff_t +_tiffSizeProc( thandle_t th ) +{ + vsi_l_offset old_off; + toff_t file_size; + VSILFILE* fp = (VSILFILE*)th; + + old_off = VSIFTellL( fp ); + (void)VSIFSeekL( fp, 0, SEEK_END ); + + file_size = (toff_t) VSIFTellL( fp ); + (void)VSIFSeekL( fp, old_off, SEEK_SET ); + + return file_size; +} + +static int +_tiffMapProc( thandle_t th, tdata_t* pbase, toff_t* psize ) +{ + (void)th; + (void)pbase; + (void)psize; + /* Unimplemented */ + return 0; +} + +static void +_tiffUnmapProc( thandle_t th, tdata_t base, toff_t size ) +{ + (void)th; + (void)base; + (void)size; + /* Unimplemented */ +} + +static char* set_conf_value(const char* key, const char* value) +{ + const char* old_val_const; + char* old_val = NULL; + old_val_const = CPLGetConfigOption(key, NULL); + if( old_val_const != NULL ) + old_val = strdup(old_val_const); + /* Prevent a directory listing to be done */ + CPLSetConfigOption(key, value); + return old_val; +} + +static void restore_conf_value(const char* key, char* old_val) +{ + CPLSetConfigOption(key, old_val); + free(old_val); +} + +typedef struct +{ + char* old_val_disable_readdir; + char* old_val_headerfile; + char* old_val_secret; + char* old_val_access; +} mapache_gdal_env_context; + +static void set_gdal_context(mapcache_cache_tiff *cache, + mapache_gdal_env_context* pcontext) +{ + memset(pcontext, 0, sizeof(mapache_gdal_env_context)); + /* Prevent a directory listing to be done */ + pcontext->old_val_disable_readdir = + set_conf_value("GDAL_DISABLE_READDIR_ON_OPEN", "YES"); + + if( cache->storage.header_file ) { + pcontext->old_val_headerfile = set_conf_value("GDAL_HTTP_HEADER_FILE", + cache->storage.header_file); + } + if( cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE ) { + pcontext->old_val_secret = set_conf_value("GS_SECRET_ACCESS_KEY", + cache->storage.u.google.secret); + pcontext->old_val_access = set_conf_value("GS_ACCESS_KEY_ID", + cache->storage.u.google.access); + } +} + +static void restore_gdal_context(mapcache_cache_tiff *cache, + mapache_gdal_env_context* pcontext) +{ + + restore_conf_value("GDAL_DISABLE_READDIR_ON_OPEN", + pcontext->old_val_disable_readdir); + if( cache->storage.header_file ) { + restore_conf_value("GDAL_HTTP_HEADER_FILE", + pcontext->old_val_headerfile); + } + if( cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE ) { + restore_conf_value("GS_SECRET_ACCESS_KEY", + pcontext->old_val_secret); + restore_conf_value("GS_ACCESS_KEY_ID", + pcontext->old_val_access); + } +} + +static int mapcache_cache_tiff_vsi_stat( + mapcache_cache_tiff *cache, + const char* name, + VSIStatBufL* pstat) +{ + mapache_gdal_env_context context; + int ret; + + set_gdal_context(cache, &context); + ret = VSIStatL(name, pstat); + restore_gdal_context(cache, &context); + + return ret; +} + +static VSILFILE* mapcache_cache_tiff_vsi_open( + mapcache_cache_tiff *cache, + const char* name, const char* mode ) +{ + mapache_gdal_env_context context; + VSILFILE* fp; + + set_gdal_context(cache, &context); + fp = VSIFOpenL(name, mode); + restore_gdal_context(cache, &context); + + return fp; +} + +static TIFF* mapcache_cache_tiff_open(mapcache_context *ctx, + mapcache_cache_tiff *cache, + const char* name, const char* mode ) +{ + char chDummy; + VSILFILE* fp; + + /* If writing or using a regular filename, then use standard */ + /* libtiff/libgeotiff I/O layer */ + if( strcmp(mode, "r") != 0 || strncmp(name, "/vsi", 4) != 0 ) + { + return MyTIFFOpen(name, mode); + } + + fp = mapcache_cache_tiff_vsi_open(cache, name, mode); + if( fp == NULL ) + return NULL; + + if( strcmp(mode, "r") == 0 ) + { + /* But then the file descriptor may point to an invalid resource */ + /* so try reading a byte from it */ + if(VSIFReadL(&chDummy, 1, 1, fp) != 1) + { + VSIFCloseL(fp); + return NULL; + } + VSIFSeekL(fp, 0, SEEK_SET); + } + + return +#ifdef USE_GEOTIFF + XTIFFClientOpen +#else + TIFFClientOpen +#endif + ( name, mode, + (thandle_t) fp, + _tiffReadProc, _tiffWriteProc, + _tiffSeekProc, _tiffCloseProc, _tiffSizeProc, + _tiffMapProc, _tiffUnmapProc ); +} + +static void CPL_STDCALL mapcache_cache_tiff_gdal_error_handler(CPLErr eErr, + int error_num, + const char* pszMsg) +{ +#ifdef DEBUG + mapcache_context *ctx = (mapcache_context *) CPLGetErrorHandlerUserData(); + ctx->log(ctx,MAPCACHE_DEBUG,"GDAL %s, %d: %s", + (eErr == CE_Failure) ? "Failure": + (eErr == CE_Warning) ? "Warning": + "Debug", + error_num, + pszMsg); +#endif +} + + +#else + +static TIFF* mapcache_cache_tiff_open(mapcache_context *ctx, + mapcache_cache_tiff *cache, + const char* name, const char* mode ) +{ + (void)ctx; + (void)cache; + return MyTIFFOpen(name, mode); + +} + +#endif /* USE_GDAL */ + +#undef MyTIFFOpen /** * \brief return filename for given tile @@ -62,10 +342,22 @@ * \param r * \private \memberof mapcache_cache_tiff */ -static void _mapcache_cache_tiff_tile_key(mapcache_context *ctx, mapcache_tile *tile, char **path) +static void _mapcache_cache_tiff_tile_key(mapcache_context *ctx, mapcache_cache_tiff *cache, mapcache_tile *tile, char **path) { - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)tile->tileset->cache; - *path = dcache->filename_template; + if( cache->storage.type == MAPCACHE_TIFF_STORAGE_REST ) { + *path = apr_pstrcat(ctx->pool, "/vsicurl/", cache->filename_template, NULL); + } + else if( cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE && + strncmp(cache->filename_template, + "https://storage.googleapis.com/", + strlen("https://storage.googleapis.com/")) == 0) { + *path = apr_pstrcat(ctx->pool, "/vsigs/", + cache->filename_template + + strlen("https://storage.googleapis.com/"), NULL); + } + else { + *path = apr_pstrdup(ctx->pool, cache->filename_template); + } /* * generic template substitutions @@ -76,14 +368,17 @@ if(strstr(*path,"{grid}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); - if(tile->dimensions && strstr(*path,"{dim}")) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->val,"/.",'#'); + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + const char *dimval = mapcache_util_str_sanitize(ctx->pool,rdim->cached_value,"/.",'#'); + char *dim_key = apr_pstrcat(ctx->pool,"{dim:",rdim->dimension->name,"}",NULL); dimstring = apr_pstrcat(ctx->pool,dimstring,"#",dimval,NULL); + if(strstr(*path,dim_key)) { + *path = mapcache_util_str_replace(ctx->pool,*path, dim_key, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -91,23 +386,23 @@ while(strstr(*path,"{z}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{z}", - apr_psprintf(ctx->pool,dcache->z_fmt,tile->z)); + apr_psprintf(ctx->pool,cache->z_fmt,tile->z)); /* * x and y replacing, when the tiff files are numbered with an increasing * x,y scheme (adjacent tiffs have x-x'=1 or y-y'=1 */ while(strstr(*path,"{div_x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{div_x}", - apr_psprintf(ctx->pool,dcache->div_x_fmt,tile->x/dcache->count_x)); + apr_psprintf(ctx->pool,cache->div_x_fmt,tile->x/cache->count_x)); while(strstr(*path,"{div_y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{div_y}", - apr_psprintf(ctx->pool,dcache->div_y_fmt,tile->y/dcache->count_y)); + apr_psprintf(ctx->pool,cache->div_y_fmt,tile->y/cache->count_y)); while(strstr(*path,"{inv_div_y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_y}", - apr_psprintf(ctx->pool,dcache->inv_div_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y)); + apr_psprintf(ctx->pool,cache->inv_div_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/cache->count_y)); while(strstr(*path,"{inv_div_x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_div_x}", - apr_psprintf(ctx->pool,dcache->inv_div_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x)); + apr_psprintf(ctx->pool,cache->inv_div_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/cache->count_x)); /* * x and y replacing, when the tiff files are numbered with the index @@ -116,25 +411,24 @@ */ while(strstr(*path,"{x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{x}", - apr_psprintf(ctx->pool,dcache->x_fmt,tile->x/dcache->count_x*dcache->count_x)); + apr_psprintf(ctx->pool,cache->x_fmt,tile->x/cache->count_x*cache->count_x)); while(strstr(*path,"{y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{y}", - apr_psprintf(ctx->pool,dcache->y_fmt,tile->y/dcache->count_y*dcache->count_y)); + apr_psprintf(ctx->pool,cache->y_fmt,tile->y/cache->count_y*cache->count_y)); while(strstr(*path,"{inv_y}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_y}", - apr_psprintf(ctx->pool,dcache->inv_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/dcache->count_y*dcache->count_y)); + apr_psprintf(ctx->pool,cache->inv_y_fmt,(tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)/cache->count_y*cache->count_y)); while(strstr(*path,"{inv_x}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_x}", - apr_psprintf(ctx->pool,dcache->inv_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/dcache->count_x*dcache->count_y)); + apr_psprintf(ctx->pool,cache->inv_x_fmt,(tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)/cache->count_x*cache->count_y)); if(!*path) { ctx->set_error(ctx,500, "failed to allocate tile key"); } } #ifdef DEBUG -static void check_tiff_format(mapcache_context *ctx, mapcache_tile *tile, TIFF *hTIFF, const char *filename) +static void check_tiff_format(mapcache_context *ctx, mapcache_cache_tiff *cache, mapcache_tile *tile, TIFF *hTIFF, const char *filename) { - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)tile->tileset->cache; uint32 imwidth,imheight,tilewidth,tileheight; int16 planarconfig,orientation; uint16 compression; @@ -192,8 +486,8 @@ * configured for the cache */ level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); if( tilewidth != tile->grid_link->grid->tile_sx || tileheight != tile->grid_link->grid->tile_sy || imwidth != tile->grid_link->grid->tile_sx * ntilesx || @@ -213,17 +507,21 @@ } #endif -static int _mapcache_cache_tiff_has_tile(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_tiff_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; TIFF *hTIFF; - mapcache_cache_tiff *dcache; - _mapcache_cache_tiff_tile_key(ctx, tile, &filename); - dcache = (mapcache_cache_tiff*)tile->tileset->cache; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; + _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } - hTIFF = MyTIFFOpen(filename,"r"); + +#ifdef USE_GDAL + CPLPushErrorHandlerEx(mapcache_cache_tiff_gdal_error_handler, ctx); +#endif + + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"r"); if(hTIFF) { do { @@ -246,15 +544,18 @@ #ifdef DEBUG - check_tiff_format(ctx,tile,hTIFF,filename); + check_tiff_format(ctx,cache,tile,hTIFF,filename); if(GC_HAS_ERROR(ctx)) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } #endif level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); /* x offset of the tile along a row */ tiff_offx = tile->x % ntilesx; @@ -268,25 +569,40 @@ if(1 != TIFFGetField( hTIFF, TIFFTAG_TILEOFFSETS, &offsets )) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } if(1 != TIFFGetField( hTIFF, TIFFTAG_TILEBYTECOUNTS, &sizes )) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } MyTIFFClose(hTIFF); if( offsets[tiff_off] > 0 && sizes[tiff_off] > 0 ) { +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_TRUE; } else { +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } } while( TIFFReadDirectory( hTIFF ) ); - return MAPCACHE_FALSE; /* TIFF only contains overviews ? */ - } else - return MAPCACHE_FALSE; + /* TIFF only contains overviews ? */ + } +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif + return MAPCACHE_FALSE; } -static void _mapcache_cache_tiff_delete(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_tiff_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { ctx->set_error(ctx,500,"TIFF cache tile deleting not implemented"); } @@ -299,14 +615,13 @@ * \private \memberof mapcache_cache_tiff * \sa mapcache_cache::tile_get() */ -static int _mapcache_cache_tiff_get(mapcache_context *ctx, mapcache_tile *tile) +static int _mapcache_cache_tiff_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { char *filename; TIFF *hTIFF = NULL; int rv; - mapcache_cache_tiff *dcache; - _mapcache_cache_tiff_tile_key(ctx, tile, &filename); - dcache = (mapcache_cache_tiff*)tile->tileset->cache; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; + _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } @@ -315,13 +630,17 @@ tile->x,tile->y,tile->z,filename); #endif - hTIFF = MyTIFFOpen(filename,"r"); +#ifdef USE_GDAL + CPLPushErrorHandlerEx(mapcache_cache_tiff_gdal_error_handler, ctx); +#endif + + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"r"); /* * we currrently have no way of knowing if the opening failed because the tif * file does not exist (which is not an error condition, as it only signals * that the requested tile does not exist in the cache), or if an other error - * that should be signaled occured (access denied, not a tiff file, etc...) + * that should be signaled occurred (access denied, not a tiff file, etc...) * * we ignore this case here and hope that further parts of the code will be * able to detect what's happening more precisely @@ -347,9 +666,12 @@ #ifdef DEBUG - check_tiff_format(ctx,tile,hTIFF,filename); + check_tiff_format(ctx,cache,tile,hTIFF,filename); if(GC_HAS_ERROR(ctx)) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } #endif @@ -359,8 +681,8 @@ * file for lower zoom levels */ level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); /* x offset of the tile along a row */ tiff_offx = tile->x % ntilesx; @@ -378,6 +700,9 @@ ctx->set_error(ctx,500,"Failed to read TIFF file \"%s\" tile offsets", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -387,6 +712,9 @@ ctx->set_error(ctx,500,"Failed to read TIFF file \"%s\" tile sizes", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -395,7 +723,7 @@ * are not zero for that index. * if not, the tiff file is sparse and is missing the requested tile */ - if( offsets[tiff_off] > 0 && sizes[tiff_off] > 0 ) { + if( offsets[tiff_off] > 0 && sizes[tiff_off] >= 2 ) { apr_file_t *f; apr_finfo_t finfo; apr_status_t ret; @@ -404,11 +732,14 @@ uint32 jpegtable_size = 0; unsigned char* jpegtable_ptr; rv = TIFFGetField( hTIFF, TIFFTAG_JPEGTABLES, &jpegtable_size, &jpegtable_ptr ); - if( rv != 1 || !jpegtable_ptr || !jpegtable_size) { + if( rv != 1 || !jpegtable_ptr || jpegtable_size < 2) { /* there is no common jpeg header in the tiff tags */ ctx->set_error(ctx,500,"Failed to read TIFF file \"%s\" jpeg table", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -416,6 +747,90 @@ * open the tiff file directly to access the jpeg image data with the given * offset */ +#ifdef USE_GDAL + if( cache->storage.type != MAPCACHE_TIFF_STORAGE_FILE ) + { + char *bufptr; + apr_off_t off; + apr_size_t bytes_to_read; + size_t bytes_read; + VSIStatBufL sStat; + VSILFILE* fp; + + fp = mapcache_cache_tiff_vsi_open(cache, filename, "r"); + if( fp == NULL ) + { + /* + * shouldn't usually happen. we managed to open the file before, + * nothing much to do except bail out. + */ + ctx->set_error(ctx,500, + "VSIFOpenL() failed on already open tiff " + "file \"%s\", giving up .... ", + filename); + MyTIFFClose(hTIFF); + CPLPopErrorHandler(); + return MAPCACHE_FAILURE; + } + + if( mapcache_cache_tiff_vsi_stat(cache, filename, &sStat) == 0 ) { + /* + * extract the file modification time. this isn't guaranteed to be the + * modification time of the actual tile, but it's the best we can do + */ + tile->mtime = sStat.st_mtime; + } + +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"tile (%d,%d,%d) => mtime = %d)", + tile->x,tile->y,tile->z,tile->mtime); +#endif + + + /* create a memory buffer to contain the jpeg data */ + tile->encoded_data = mapcache_buffer_create( + (jpegtable_size+sizes[tiff_off]-4),ctx->pool); + + /* + * copy the jpeg header to the beginning of the memory buffer, + * omitting the last 2 bytes + */ + memcpy(tile->encoded_data->buf,jpegtable_ptr,(jpegtable_size-2)); + + /* advance the data pointer to after the header data */ + bufptr = ((char *)tile->encoded_data->buf) + (jpegtable_size-2); + + + /* go to the specified offset in the tiff file, plus 2 bytes */ + off = offsets[tiff_off]+2; + VSIFSeekL(fp, (vsi_l_offset)off, SEEK_SET); + + /* + * copy the jpeg body at the end of the memory buffer, accounting + * for the two bytes we omitted in the previous step + */ + bytes_to_read = sizes[tiff_off]-2; + bytes_read = VSIFReadL(bufptr, 1, bytes_to_read, fp); + + /* check we have correctly read the requested number of bytes */ + if(bytes_to_read != bytes_read) { + ctx->set_error(ctx,500,"failed to read jpeg body in \"%s\".\ + (read %d of %d bytes)", + filename,(int)bytes_read,(int)sizes[tiff_off]-2); + VSIFCloseL(fp); + MyTIFFClose(hTIFF); + CPLPopErrorHandler(); + return MAPCACHE_FAILURE; + } + + tile->encoded_data->size = (jpegtable_size+sizes[tiff_off]-4); + + VSIFCloseL(fp); + CPLPopErrorHandler(); + return MAPCACHE_SUCCESS; + } + else +#endif if((ret=apr_file_open(&f, filename, APR_FOPEN_READ|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,APR_OS_DEFAULT, ctx->pool)) == APR_SUCCESS) { @@ -441,7 +856,7 @@ memcpy(tile->encoded_data->buf,jpegtable_ptr,(jpegtable_size-2)); /* advance the data pointer to after the header data */ - bufptr = tile->encoded_data->buf + (jpegtable_size-2); + bufptr = ((char *)tile->encoded_data->buf) + (jpegtable_size-2); /* go to the specified offset in the tiff file, plus 2 bytes */ @@ -459,7 +874,11 @@ if(bytes != sizes[tiff_off]-2) { ctx->set_error(ctx,500,"failed to read jpeg body in \"%s\".\ (read %d of %d bytes)", filename,bytes,sizes[tiff_off]-2); + apr_file_close(f); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -468,6 +887,9 @@ /* finalize and cleanup */ apr_file_close(f); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_SUCCESS; } else { /* shouldn't usually happen. we managed to open the file with TIFFOpen, @@ -477,11 +899,17 @@ ctx->set_error(ctx,500,"apr_file_open failed on already open tiff file \"%s\", giving up .... ", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } } else { /* sparse tiff file without the requested tile */ MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_CACHE_MISS; } } /* loop through the tiff directories if there are multiple ones */ @@ -493,8 +921,10 @@ * does the file only contain overviews? */ MyTIFFClose(hTIFF); - return MAPCACHE_CACHE_MISS; } +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif /* failed to open tiff file */ return MAPCACHE_CACHE_MISS; } @@ -508,17 +938,16 @@ * \private \memberof mapcache_cache_tiff * \sa mapcache_cache::tile_set() */ -static void _mapcache_cache_tiff_set(mapcache_context *ctx, mapcache_tile *tile) +static void _mapcache_cache_tiff_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { #ifdef USE_TIFF_WRITE char *filename; TIFF *hTIFF = NULL; int rv; + void *lock; int create; - char errmsg[120]; - mapcache_cache_tiff *dcache; + mapcache_cache_tiff *cache; mapcache_image_format_jpeg *format; - char *hackptr1,*hackptr2; int tilew; int tileh; unsigned char *rgb; @@ -530,9 +959,14 @@ int tiff_offx, tiff_offy; /* the x and y offset of the tile inside the tiff image */ int tiff_off; /* the index of the tile inside the list of tiles of the tiff image */ - _mapcache_cache_tiff_tile_key(ctx, tile, &filename); - dcache = (mapcache_cache_tiff*)tile->tileset->cache; - format = (mapcache_image_format_jpeg*) dcache->format; + cache = (mapcache_cache_tiff*)pcache; + if( cache->storage.type != MAPCACHE_TIFF_STORAGE_FILE ) + { + ctx->set_error(ctx,500,"tiff cache %s is read-only\n",pcache->name); + return; + } + _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); + format = (mapcache_image_format_jpeg*) cache->format; if(GC_HAS_ERROR(ctx)) { return; } @@ -544,27 +978,8 @@ /* * create the directory where the tiff file will be stored */ - - /* find the location of the last '/' in the string */ - hackptr2 = hackptr1 = filename; - while(*hackptr1) { - if(*hackptr1 == '/') - hackptr2 = hackptr1; - hackptr1++; - } - *hackptr2 = '\0'; - - if(APR_SUCCESS != (rv = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - /* - * apr_dir_make_recursive sometimes sends back this error, although it should not. - * ignore this one - */ - if(!APR_STATUS_IS_EEXIST(rv)) { - ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(rv,errmsg,120)); - return; - } - } - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); tilew = tile->grid_link->grid->tile_sx; tileh = tile->grid_link->grid->tile_sy; @@ -592,15 +1007,15 @@ * aquire a lock on the tiff file. */ - while(mapcache_lock_or_wait_for_resource(ctx,filename) == MAPCACHE_FALSE); + while(mapcache_lock_or_wait_for_resource(ctx,(cache->locker?cache->locker:ctx->config->locker),filename, &lock) == MAPCACHE_FALSE); /* check if the tiff file exists already */ rv = apr_stat(&finfo,filename,0,ctx->pool); if(!APR_STATUS_IS_ENOENT(rv)) { - hTIFF = MyTIFFOpen(filename,"r+"); + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"r+"); create = 0; } else { - hTIFF = MyTIFFOpen(filename,"w+"); + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"w+"); create = 1; } if(!hTIFF) { @@ -615,11 +1030,12 @@ * file for lower zoom levels */ level = tile->grid_link->grid->levels[tile->z]; - ntilesx = MAPCACHE_MIN(dcache->count_x, level->maxx); - ntilesy = MAPCACHE_MIN(dcache->count_y, level->maxy); + ntilesx = MAPCACHE_MIN(cache->count_x, level->maxx); + ntilesy = MAPCACHE_MIN(cache->count_y, level->maxy); if(create) { #ifdef USE_GEOTIFF - double adfPixelScale[3], adfTiePoints[6], bbox[4]; + double adfPixelScale[3], adfTiePoints[6]; + mapcache_extent bbox; GTIF *gtif; int x,y; #endif @@ -679,16 +1095,16 @@ /* top left tile x,y */ - x = (tile->x / dcache->count_x)*(dcache->count_x); - y = (tile->y / dcache->count_y)*(dcache->count_y) + ntilesy - 1; + x = (tile->x / cache->count_x)*(cache->count_x); + y = (tile->y / cache->count_y)*(cache->count_y) + ntilesy - 1; - mapcache_grid_get_extent(ctx, tile->grid_link->grid, - x,y,tile->z,bbox); + mapcache_grid_get_tile_extent(ctx, tile->grid_link->grid, + x,y,tile->z,&bbox); adfTiePoints[0] = 0.0; adfTiePoints[1] = 0.0; adfTiePoints[2] = 0.0; - adfTiePoints[3] = bbox[0]; - adfTiePoints[4] = bbox[3]; + adfTiePoints[3] = bbox.minx; + adfTiePoints[4] = bbox.maxy; adfTiePoints[5] = 0.0; TIFFSetField( hTIFF, TIFFTAG_GEOTIEPOINTS, 6, adfTiePoints ); } @@ -737,7 +1153,7 @@ close_tiff: if(hTIFF) MyTIFFClose(hTIFF); - mapcache_unlock_resource(ctx,filename); + mapcache_unlock_resource(ctx,cache->locker?cache->locker:ctx->config->locker, lock); #else ctx->set_error(ctx,500,"tiff write support disabled by default"); #endif @@ -747,76 +1163,73 @@ /** * \private \memberof mapcache_cache_tiff */ -static void _mapcache_cache_tiff_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_tiff_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *pcache, mapcache_cfg *config) { ezxml_t cur_node; - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)cache; - ezxml_t xcount; - ezxml_t ycount; - ezxml_t xformat; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; char * format_name; mapcache_image_format *pformat; if ((cur_node = ezxml_child(node,"template")) != NULL) { char *fmt; - dcache->filename_template = apr_pstrdup(ctx->pool,cur_node->txt); + cache->filename_template = apr_pstrdup(ctx->pool,cur_node->txt); fmt = (char*)ezxml_attr(cur_node,"x_fmt"); if(fmt && *fmt) { - dcache->x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"y_fmt"); if(fmt && *fmt) { - dcache->y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->y_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"z_fmt"); if(fmt && *fmt) { - dcache->z_fmt = apr_pstrdup(ctx->pool,fmt); + cache->z_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_x_fmt"); if(fmt && *fmt) { - dcache->inv_x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_y_fmt"); if(fmt && *fmt) { - dcache->inv_y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_y_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"div_x_fmt"); if(fmt && *fmt) { - dcache->div_x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->div_x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"div_y_fmt"); if(fmt && *fmt) { - dcache->div_y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->div_y_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_div_x_fmt"); if(fmt && *fmt) { - dcache->inv_div_x_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_div_x_fmt = apr_pstrdup(ctx->pool,fmt); } fmt = (char*)ezxml_attr(cur_node,"inv_div_y_fmt"); if(fmt && *fmt) { - dcache->inv_div_y_fmt = apr_pstrdup(ctx->pool,fmt); + cache->inv_div_y_fmt = apr_pstrdup(ctx->pool,fmt); } } - xcount = ezxml_child(node,"xcount"); - if(xcount && xcount->txt && *xcount->txt) { + cur_node = ezxml_child(node,"xcount"); + if(cur_node && cur_node->txt && *cur_node->txt) { char *endptr; - dcache->count_x = (int)strtol(xcount->txt,&endptr,10); + cache->count_x = (int)strtol(cur_node->txt,&endptr,10); if(*endptr != 0) { - ctx->set_error(ctx,400,"failed to parse xcount value %s for tiff cache %s", xcount->txt,cache->name); + ctx->set_error(ctx,400,"failed to parse xcount value %s for tiff cache %s", cur_node->txt,pcache->name); return; } } - ycount = ezxml_child(node,"ycount"); - if(ycount && ycount->txt && *ycount->txt) { + cur_node = ezxml_child(node,"ycount"); + if(cur_node && cur_node->txt && *cur_node->txt) { char *endptr; - dcache->count_y = (int)strtol(ycount->txt,&endptr,10); + cache->count_y = (int)strtol(cur_node->txt,&endptr,10); if(*endptr != 0) { - ctx->set_error(ctx,400,"failed to parse ycount value %s for tiff cache %s", ycount->txt,cache->name); + ctx->set_error(ctx,400,"failed to parse ycount value %s for tiff cache %s", cur_node->txt,pcache->name); return; } } - xformat = ezxml_child(node,"format"); - if(xformat && xformat->txt && *xformat->txt) { - format_name = xformat->txt; + cur_node = ezxml_child(node,"format"); + if(cur_node && cur_node->txt && *cur_node->txt) { + format_name = cur_node->txt; } else { format_name = "JPEG"; } @@ -824,33 +1237,146 @@ config,format_name); if(!pformat) { ctx->set_error(ctx,500,"TIFF cache %s references unknown image format %s", - cache->name, format_name); + pcache->name, format_name); return; } if(pformat->type != GC_JPEG) { ctx->set_error(ctx,500,"TIFF cache %s can only reference a JPEG image format", - cache->name); + pcache->name); return; } - dcache->format = (mapcache_image_format_jpeg*)pformat; + cache->format = (mapcache_image_format_jpeg*)pformat; + + cur_node = ezxml_child(node,"locker"); + if(cur_node) { + mapcache_config_parse_locker(ctx, cur_node, &cache->locker); + } + + cache->storage.type = MAPCACHE_TIFF_STORAGE_FILE; + cur_node = ezxml_child(node,"storage"); + if (cur_node) { + ezxml_t child_node; + const char *type = ezxml_attr(cur_node,"type"); + if( !type ) { + ctx->set_error(ctx,400, + " with no \"type\" attribute in cache (%s)", + pcache->name); + return; + } + + if( strcmp(type, "rest") == 0 ) { + cache->storage.type = MAPCACHE_TIFF_STORAGE_REST; + } + else if( strcmp(type, "google") == 0 ) { + cache->storage.type = MAPCACHE_TIFF_STORAGE_GOOGLE; + if ((child_node = ezxml_child(cur_node,"access")) != NULL) { + cache->storage.u.google.access = + apr_pstrdup(ctx->pool, child_node->txt); + } else if ( getenv("GS_ACCESS_KEY_ID")) { + cache->storage.u.google.access = + apr_pstrdup(ctx->pool,getenv("GS_ACCESS_KEY_ID")); + } else { + ctx->set_error(ctx,400, + "google storage in cache (%s) is missing " + "required child", pcache->name); + return; + } + if ((child_node = ezxml_child(cur_node,"secret")) != NULL) { + cache->storage.u.google.secret = + apr_pstrdup(ctx->pool, child_node->txt); + } else if ( getenv("GS_SECRET_ACCESS_KEY")) { + cache->storage.u.google.access = + apr_pstrdup(ctx->pool,getenv("GS_SECRET_ACCESS_KEY")); + } else { + ctx->set_error(ctx,400, + "google storage in cache (%s) is missing " + "required child", pcache->name); + return; + } + } + else { + ctx->set_error(ctx, 400, "unknown cache type %s for cache \"%s\"", + type, pcache->name); + return; + } + + if ((child_node = ezxml_child(cur_node,"connection_timeout")) != NULL) { + char *endptr; + cache->storage.connection_timeout = (int)strtol(child_node->txt,&endptr,10); + if(*endptr != 0 || cache->storage.connection_timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache " + "\"%s\" (positive integer expected)", + child_node->txt); + return; + } + } else { + cache->storage.connection_timeout = 30; + } + + if ((child_node = ezxml_child(cur_node,"timeout")) != NULL) { + char *endptr; + cache->storage.timeout = (int)strtol(child_node->txt,&endptr,10); + if(*endptr != 0 || cache->storage.timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache \"%s\" " + "(positive integer expected)", + child_node->txt); + return; + } + } else { + cache->storage.timeout = 120; + } + + if ((child_node = ezxml_child(cur_node,"header_file")) != NULL) { + cache->storage.header_file = apr_pstrdup(ctx->pool, child_node->txt); + } + + } + } /** * \private \memberof mapcache_cache_tiff */ -static void _mapcache_cache_tiff_configuration_post_config(mapcache_context *ctx, mapcache_cache *cache, +static void _mapcache_cache_tiff_configuration_post_config(mapcache_context *ctx, mapcache_cache *pcache, mapcache_cfg *cfg) { - mapcache_cache_tiff *dcache = (mapcache_cache_tiff*)cache; + mapcache_cache_tiff *cache = (mapcache_cache_tiff*)pcache; /* check all required parameters are configured */ - if((!dcache->filename_template || !strlen(dcache->filename_template))) { - ctx->set_error(ctx, 400, "tiff cache %s has no template pattern",dcache->cache.name); + if((!cache->filename_template || !strlen(cache->filename_template))) { + ctx->set_error(ctx, 400, "tiff cache %s has no template pattern",cache->cache.name); + return; + } + if(cache->count_x <= 0 || cache->count_y <= 0) { + ctx->set_error(ctx, 400, "tiff cache %s has invalid count (%d,%d)",cache->count_x,cache->count_y); return; } - if(dcache->count_x <= 0 || dcache->count_y <= 0) { - ctx->set_error(ctx, 400, "tiff cache %s has invalid count (%d,%d)",dcache->count_x,dcache->count_y); + +#ifdef USE_GDAL + if(cache->storage.type == MAPCACHE_TIFF_STORAGE_REST && + strncmp(cache->filename_template, "http://", 6) != 0 && + strncmp(cache->filename_template, "https://", 7) != 0 ) { + ctx->set_error(ctx, 400, "tiff cache %s template pattern should begin with http:// or https://",cache->cache.name); return; } + + if(cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE && + strncmp(cache->filename_template, "https://storage.googleapis.com/", + strlen("https://storage.googleapis.com/")) != 0 && + strncmp(cache->filename_template, "/vsigs/", + strlen("/vsigs/")) != 0 ) { + ctx->set_error(ctx, 400, "tiff cache %s template pattern should begin " + "with https://storage.googleapis.com/ or /vsigs/",cache->cache.name); + return; + } +#else + if(cache->storage.type != MAPCACHE_TIFF_STORAGE_FILE ) + { + ctx->set_error(ctx, 400, "tiff cache %s cannot use a network based " + "storage due to mising GDAL dependency", + cache->cache.name); + return; + } +#endif } /** @@ -865,10 +1391,10 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_TIFF; - cache->cache.tile_delete = _mapcache_cache_tiff_delete; - cache->cache.tile_get = _mapcache_cache_tiff_get; - cache->cache.tile_exists = _mapcache_cache_tiff_has_tile; - cache->cache.tile_set = _mapcache_cache_tiff_set; + cache->cache._tile_delete = _mapcache_cache_tiff_delete; + cache->cache._tile_get = _mapcache_cache_tiff_get; + cache->cache._tile_exists = _mapcache_cache_tiff_has_tile; + cache->cache._tile_set = _mapcache_cache_tiff_set; cache->cache.configuration_post_config = _mapcache_cache_tiff_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_tiff_configuration_parse_xml; cache->count_x = 10; @@ -884,6 +1410,13 @@ return (mapcache_cache*)cache; } +#else + +mapcache_cache* mapcache_cache_tiff_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"TIFF support not compiled in this version"); + return NULL; +} + #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.2.1/lib/cache_tokyocabinet.c mapcache-1.6.1/lib/cache_tokyocabinet.c --- mapcache-1.2.1/lib/cache_tokyocabinet.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/cache_tokyocabinet.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,9 +27,9 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#include "mapcache.h" #ifdef USE_TC -#include "mapcache.h" #include #include #include @@ -43,6 +43,14 @@ #include #endif +typedef struct mapcache_cache_tc mapcache_cache_tc; +struct mapcache_cache_tc { + mapcache_cache cache; + char *basedir; + char *key_template; + mapcache_context *ctx; +}; +mapcache_cache *mapcache_cache_tc_create(mapcache_context *ctx); struct tc_conn { TCBDB *bdb; @@ -216,6 +224,13 @@ return (mapcache_cache*)cache; } +#else + +mapcache_cache* mapcache_cache_tc_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"TokyoCabinet support not compiled in this version"); + return NULL; +} + #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.2.1/lib/configuration.c mapcache-1.6.1/lib/configuration.c --- mapcache-1.2.1/lib/configuration.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/configuration.c 2017-09-29 21:31:13.000000000 +0000 @@ -35,47 +35,12 @@ void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi) { - apr_dir_t *lockdir; - apr_status_t rv; - char errmsg[120]; char *url; mapcache_configuration_parse_xml(ctx,filename,config); - GC_CHECK_ERROR(ctx); - if(!config->lockdir || !strlen(config->lockdir)) { - config->lockdir = apr_pstrdup(ctx->pool, "/tmp"); - } - rv = apr_dir_open(&lockdir,config->lockdir,ctx->pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500, "failed to open lock directory %s: %s" - ,config->lockdir,apr_strerror(rv,errmsg,120)); - return; - } - - /* only remove lockfiles if we're not in cgi mode */ - if(!cgi) { - apr_finfo_t finfo; - while ((apr_dir_read(&finfo, APR_FINFO_DIRENT|APR_FINFO_TYPE|APR_FINFO_NAME, lockdir)) == APR_SUCCESS) { - if(finfo.filetype == APR_REG) { - if(!strncmp(finfo.name, MAPCACHE_LOCKFILE_PREFIX, strlen(MAPCACHE_LOCKFILE_PREFIX))) { - ctx->log(ctx,MAPCACHE_WARN,"found old lockfile %s/%s, deleting it",config->lockdir, - finfo.name); - rv = apr_file_remove(apr_psprintf(ctx->pool,"%s/%s",config->lockdir, finfo.name),ctx->pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500, "failed to remove lockfile %s: %s",finfo.name,apr_strerror(rv,errmsg,120)); - return; - } - - } - - } - } - } - apr_dir_close(lockdir); - /* if we were suppplied with an onlineresource, make sure it ends with a / */ if(NULL != (url = (char*)apr_table_get(config->metadata,"url"))) { char *urlend = url + strlen(url)-1; @@ -170,12 +135,12 @@ mapcache_imageio_create_png_q_format(pool,"PNG8",MAPCACHE_COMPRESSION_FAST,256), "PNG8"); mapcache_configuration_add_image_format(cfg, - mapcache_imageio_create_jpeg_format(pool,"JPEG",90,MAPCACHE_PHOTOMETRIC_YCBCR), + mapcache_imageio_create_jpeg_format(pool,"JPEG",90,MAPCACHE_PHOTOMETRIC_YCBCR,MAPCACHE_OPTIMIZE_YES), "JPEG"); mapcache_configuration_add_image_format(cfg, mapcache_imageio_create_mixed_format(pool,"mixed", mapcache_configuration_get_image_format(cfg,"PNG"), - mapcache_configuration_get_image_format(cfg,"JPEG")), + mapcache_configuration_get_image_format(cfg,"JPEG"), 255), "mixed"); cfg->default_image_format = mapcache_configuration_get_image_format(cfg,"mixed"); cfg->reporting = MAPCACHE_REPORT_MSG; @@ -254,8 +219,6 @@ } mapcache_configuration_add_grid(cfg,grid,"g"); - /* default retry interval is 1/100th of a second, i.e. 10000 microseconds */ - cfg->lock_retry_interval = 10000; cfg->loglevel = MAPCACHE_WARN; cfg->autoreload = 0; diff -Nru mapcache-1.2.1/lib/configuration_xml.c mapcache-1.6.1/lib/configuration_xml.c --- mapcache-1.2.1/lib/configuration_xml.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/configuration_xml.c 2017-09-29 21:31:13.000000000 +0000 @@ -47,47 +47,6 @@ } } -void parseTimeDimension(mapcache_context *ctx, ezxml_t node, mapcache_tileset *tileset) { - const char *attr = NULL; - if(tileset->timedimension) { - ctx->set_error(ctx,400,"tileset \"%s\" can only have a single ", tileset->name); - return; - } - attr = ezxml_attr(node,"type"); - if(attr && *attr) { - if(!strcmp(attr,"sqlite")) { -#ifdef USE_SQLITE - tileset->timedimension = mapcache_timedimension_sqlite_create(ctx->pool); -#else - ctx->set_error(ctx,400, "failed to add sqlite timedimension: Sqlite support is not available on this build"); - return; -#endif - } else { - ctx->set_error(ctx,400,"unknown \"type\" attribute \"%s\" for %s's . Expecting one of (sqlite)",attr,tileset->name); - return; - } - - } else { - ctx->set_error(ctx,400,"missing \"type\" attribute for %s's ",tileset->name); - return; - } - attr = ezxml_attr(node,"name"); - if(attr && *attr) { - tileset->timedimension->key = apr_pstrdup(ctx->pool,attr); - } else { - tileset->timedimension->key = apr_pstrdup(ctx->pool,"TIME"); - } - - attr = ezxml_attr(node,"default"); - if(attr && *attr) { - tileset->timedimension->default_value = apr_pstrdup(ctx->pool,attr); - } else { - ctx->set_error(ctx,400,"no \"default\" attribute for %s",tileset->timedimension->key); - return; - } - tileset->timedimension->configuration_parse_xml(ctx,tileset->timedimension,node); -} - void parseDimensions(mapcache_context *ctx, ezxml_t node, mapcache_tileset *tileset) { ezxml_t dimension_node; @@ -107,15 +66,13 @@ if(type && *type) { if(!strcmp(type,"values")) { - dimension = mapcache_dimension_values_create(ctx->pool); + dimension = mapcache_dimension_values_create(ctx,ctx->pool); } else if(!strcmp(type,"regex")) { - dimension = mapcache_dimension_regex_create(ctx->pool); - } else if(!strcmp(type,"intervals")) { - dimension = mapcache_dimension_intervals_create(ctx->pool); + dimension = mapcache_dimension_regex_create(ctx,ctx->pool); + } else if(!strcmp(type,"sqlite")) { + dimension = mapcache_dimension_sqlite_create(ctx,ctx->pool); } else if(!strcmp(type,"time")) { - ctx->set_error(ctx,501,"time dimension type not implemented yet"); - return; - dimension = mapcache_dimension_time_create(ctx->pool); + dimension = mapcache_dimension_time_create(ctx,ctx->pool); } else { ctx->set_error(ctx,400,"unknown dimension type \"%s\"",type); return; @@ -124,19 +81,21 @@ ctx->set_error(ctx,400, "mandatory attribute \"type\" not found in "); return; } + GC_CHECK_ERROR(ctx); dimension->name = apr_pstrdup(ctx->pool,name); if(unit && *unit) { dimension->unit = apr_pstrdup(ctx->pool,unit); } - + if(default_value && *default_value) { dimension->default_value = apr_pstrdup(ctx->pool,default_value); } else { ctx->set_error(ctx,400,"dimension \"%s\" has no \"default\" attribute",dimension->name); return; } + dimension->configuration_parse_xml(ctx,dimension,dimension_node); GC_CHECK_ERROR(ctx); @@ -147,7 +106,51 @@ ctx->set_error(ctx, 400, " for tileset \"%s\" has no dimensions defined (expecting children)",tileset->name); return; } + tileset->dimensions = dimensions; + dimension_node = ezxml_child(node,"store_assemblies"); + if(dimension_node && dimension_node->txt) { + if(!strcmp(dimension_node->txt,"false")) { + tileset->store_dimension_assemblies = 0; + } else if(strcmp(dimension_node->txt,"true")) { + ctx->set_error(ctx,400,"failed to parse (%s), expecting \"true\" or \"false\"",dimension_node->txt); + return; + } + } + + dimension_node = ezxml_child(node,"assembly_type"); + if(dimension_node) { + if(!strcmp(dimension_node->txt,"stack")) { + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_STACK; + } else if(!strcmp(dimension_node->txt,"animate")) { + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_ANIMATE; + ctx->set_error(ctx,400,"animate dimension assembly mode not implemented"); + return; + } else if(strcmp(dimension_node->txt,"none")) { + ctx->set_error(ctx,400,"unknown dimension assembly mode (%s). Can be one of \"stack\" or \"none\"",dimension_node->txt); + return; + } else { + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_NONE; + } + } + + /* should we create subdimensions from source if not found in cache. + e.g. if dimension=mosaic returns dimension=val1,val2,val3 should we + query the wms source with dimension=val1 , dimension=val2 and/or + dimension=val3 if they are not found in cache */ + dimension_node = ezxml_child(node,"subdimensions_read_only"); + if(dimension_node) { + if(tileset->dimension_assembly_type == MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + ctx->set_error(ctx,400," used on a tileset with no set, which makes no sense"); + return; + } + if(dimension_node && dimension_node->txt && !strcmp(dimension_node->txt,"true")) { + tileset->subdimension_read_only = 1; + } else if(strcmp(dimension_node->txt,"false")) { + ctx->set_error(ctx,400,"failed to parse (%s), expecting \"true\" or \"false\"",dimension_node->txt); + return; + } + } } void parseGrid(mapcache_context *ctx, ezxml_t node, mapcache_cfg *config) @@ -333,14 +336,14 @@ source = NULL; if(!strcmp(type,"wms")) { source = mapcache_source_wms_create(ctx); -#ifdef USE_MAPSERVER } else if(!strcmp(type,"mapserver")) { source = mapcache_source_mapserver_create(ctx); -#endif } else if(!strcmp(type,"gdal")) { source = mapcache_source_gdal_create(ctx); } else if(!strcmp(type,"dummy")) { source = mapcache_source_dummy_create(ctx); + } else if(!strcmp(type,"fallback")) { + source = mapcache_source_fallback_create(ctx); } else { ctx->set_error(ctx, 400, "unknown source type %s for source \"%s\"", type, name); return; @@ -355,8 +358,22 @@ parseMetadata(ctx, cur_node, source->metadata); GC_CHECK_ERROR(ctx); } + if ((cur_node = ezxml_child(node,"retries")) != NULL) { + source->retry_count = atoi(cur_node->txt); + if(source->retry_count > 10) { + ctx->set_error(ctx,400,"source (%s) count of %d is unreasonably large. max is 10", source->name, source->retry_count); + return; + } + } + if ((cur_node = ezxml_child(node,"retry_delay")) != NULL) { + source->retry_delay = (double)atof(cur_node->txt); + if(source->retry_delay < 0) { + ctx->set_error(ctx,400,"source (%s) retry delay of %f must be positive",source->name, source->retry_delay); + return; + } + } - source->configuration_parse_xml(ctx,node,source); + source->configuration_parse_xml(ctx,node,source, config); GC_CHECK_ERROR(ctx); source->configuration_check(ctx,config,source); GC_CHECK_ERROR(ctx); @@ -387,6 +404,8 @@ compression = MAPCACHE_COMPRESSION_FAST; } else if(!strcmp(cur_node->txt, "best")) { compression = MAPCACHE_COMPRESSION_BEST; + } else if(!strcmp(cur_node->txt, "none")) { + compression = MAPCACHE_COMPRESSION_DISABLE; } else { ctx->set_error(ctx, 400, "unknown compression type %s for format \"%s\"", cur_node->txt, name); return; @@ -413,6 +432,7 @@ } } else if(!strcmp(type,"JPEG")) { int quality = 95; + int optimize = TRUE; mapcache_photometric photometric = MAPCACHE_PHOTOMETRIC_YCBCR; if ((cur_node = ezxml_child(node,"quality")) != NULL) { char *endptr; @@ -436,10 +456,24 @@ return; } } + if ((cur_node = ezxml_child(node,"optimize")) != NULL) { + if(cur_node->txt && !strcasecmp(cur_node->txt,"false")) + optimize = MAPCACHE_OPTIMIZE_NO; + else if(cur_node->txt && !strcasecmp(cur_node->txt,"true")) + optimize = MAPCACHE_OPTIMIZE_YES; + else if(cur_node->txt && !strcasecmp(cur_node->txt,"arithmetic")) + optimize = MAPCACHE_OPTIMIZE_ARITHMETIC; + else { + ctx->set_error(ctx,500,"failed to parse jpeg format %s optimize %s. expecting true, false or arithmetic", + name,cur_node->txt); + return; + } + } format = mapcache_imageio_create_jpeg_format(ctx->pool, - name,quality,photometric); + name,quality,photometric,optimize); } else if(!strcasecmp(type,"MIXED")) { mapcache_image_format *transparent=NULL, *opaque=NULL; + unsigned int alpha_cutoff=255; if ((cur_node = ezxml_child(node,"transparent")) != NULL) { transparent = mapcache_configuration_get_image_format(config,cur_node->txt); } @@ -458,7 +492,10 @@ name,cur_node->txt,cur_node->txt); return; } - format = mapcache_imageio_create_mixed_format(ctx->pool,name,transparent, opaque); + if ((cur_node = ezxml_child(node,"alpha_cutoff")) != NULL) { + alpha_cutoff = atoi(cur_node->txt); + } + format = mapcache_imageio_create_mixed_format(ctx->pool,name,transparent, opaque, alpha_cutoff); } else { ctx->set_error(ctx, 400, "unknown format type %s for format \"%s\"", type, name); return; @@ -477,6 +514,7 @@ { char *name = NULL, *type = NULL; mapcache_cache *cache = NULL; + ezxml_t cur_node; name = (char*)ezxml_attr(node,"name"); type = (char*)ezxml_attr(node,"type"); if(!name || !strlen(name)) { @@ -496,58 +534,63 @@ } if(!strcmp(type,"disk")) { cache = mapcache_cache_disk_create(ctx); + } else if(!strcmp(type,"fallback")) { + cache = mapcache_cache_fallback_create(ctx); + } else if(!strcmp(type,"multitier")) { + cache = mapcache_cache_multitier_create(ctx); + } else if(!strcmp(type,"composite")) { + cache = mapcache_cache_composite_create(ctx); + } else if(!strcmp(type,"rest")) { + cache = mapcache_cache_rest_create(ctx); + } else if(!strcmp(type,"s3")) { + cache = mapcache_cache_s3_create(ctx); + } else if(!strcmp(type,"azure")) { + cache = mapcache_cache_azure_create(ctx); + } else if(!strcmp(type,"google")) { + cache = mapcache_cache_google_create(ctx); } else if(!strcmp(type,"bdb")) { -#ifdef USE_BDB cache = mapcache_cache_bdb_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": Berkeley DB support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"tokyocabinet")) { -#ifdef USE_TC cache = mapcache_cache_tc_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": Tokyo Cabinet support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"sqlite3")) { -#ifdef USE_SQLITE cache = mapcache_cache_sqlite_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": sqlite support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"mbtiles")) { -#ifdef USE_SQLITE cache = mapcache_cache_mbtiles_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": sqlite support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"memcache")) { -#ifdef USE_MEMCACHE cache = mapcache_cache_memcache_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": memcache support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"tiff")) { -#ifdef USE_TIFF cache = mapcache_cache_tiff_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": tiff support is not available on this build",name); - return; -#endif + } else if(!strcmp(type,"couchbase")) { + cache = mapcache_cache_couchbase_create(ctx); + } else if(!strcmp(type,"riak")) { + cache = mapcache_cache_riak_create(ctx); } else { ctx->set_error(ctx, 400, "unknown cache type %s for cache \"%s\"", type, name); return; } + GC_CHECK_ERROR(ctx); if(cache == NULL) { ctx->set_error(ctx, 400, "failed to parse cache \"%s\"", name); return; } cache->name = name; + if ((cur_node = ezxml_child(node,"retries")) != NULL) { + cache->retry_count = atoi(cur_node->txt); + if(cache->retry_count > 10) { + ctx->set_error(ctx,400,"cache (%s) count of %d is unreasonably large. max is 10", cache->name, cache->retry_count); + return; + } + } + if ((cur_node = ezxml_child(node,"retry_delay")) != NULL) { + cache->retry_delay = (double)atof(cur_node->txt); + if(cache->retry_delay < 0) { + ctx->set_error(ctx,400,"cache (%s) retry delay of %f must be positive",cache->name, cache->retry_delay); + return; + } + } + + cache->configuration_parse_xml(ctx,node,cache,config); GC_CHECK_ERROR(ctx); mapcache_configuration_add_cache(config,cache,name); @@ -632,6 +675,7 @@ gridlink->maxz = grid->nlevels; gridlink->grid_limits = (mapcache_extent_i*)apr_pcalloc(ctx->pool,grid->nlevels*sizeof(mapcache_extent_i)); gridlink->outofzoom_strategy = MAPCACHE_OUTOFZOOM_NOTCONFIGURED; + gridlink->intermediate_grids = apr_array_make(ctx->pool,1,sizeof(mapcache_grid_link*)); restrictedExtent = (char*)ezxml_attr(cur_node,"restricted_extent"); if(restrictedExtent) { @@ -667,6 +711,11 @@ return; } } + sTolerance = (char*)ezxml_attr(cur_node,"use_wms_intermediate_resolutions"); + if(sTolerance && !strcmp(sTolerance,"true")) { + mapcache_grid_link *intermediate_gridlink = apr_pcalloc(ctx->pool,sizeof(mapcache_grid_link)); + APR_ARRAY_PUSH(gridlink->intermediate_grids,mapcache_grid_link*) = intermediate_gridlink; + } mapcache_grid_compute_limits(grid,extent,gridlink->grid_limits,tolerance); @@ -741,6 +790,40 @@ if(!havewgs84bbox && !strcasecmp(grid->srs,"EPSG:4326")) { tileset->wgs84bbox = *extent; } + + if(gridlink->intermediate_grids->nelts > 0) { + double factor = 0.5, unitheight,unitwidth; + int i; + mapcache_grid_link *igl = APR_ARRAY_IDX(gridlink->intermediate_grids, 0, mapcache_grid_link*); + igl->restricted_extent = gridlink->restricted_extent; + igl->minz = gridlink->minz; + igl->max_cached_zoom = gridlink->max_cached_zoom - 1; + igl->maxz = gridlink->maxz - 1; + igl->outofzoom_strategy = gridlink->outofzoom_strategy; + igl->grid = mapcache_grid_create(ctx->pool); + igl->grid->extent = gridlink->grid->extent; + igl->grid->name = apr_psprintf(ctx->pool,"%s_intermediate_%g",gridlink->grid->name,factor); + igl->grid->nlevels = gridlink->grid->nlevels - 1; + igl->grid->origin = gridlink->grid->origin; + igl->grid->srs = gridlink->grid->srs; + igl->grid->srs_aliases = gridlink->grid->srs_aliases; + igl->grid->unit = gridlink->grid->unit; + igl->grid->tile_sx = gridlink->grid->tile_sx + gridlink->grid->tile_sx * factor; + igl->grid->tile_sy = gridlink->grid->tile_sy + gridlink->grid->tile_sy * factor; + igl->grid->levels = (mapcache_grid_level**)apr_pcalloc(ctx->pool, igl->grid->nlevels*sizeof(mapcache_grid_level*)); + for(i=0; igrid->nlevels; i++) { + mapcache_grid_level *level = (mapcache_grid_level*)apr_pcalloc(ctx->pool,sizeof(mapcache_grid_level)); + level->resolution = gridlink->grid->levels[i]->resolution + (gridlink->grid->levels[i+1]->resolution - gridlink->grid->levels[i]->resolution) * factor; + unitheight = igl->grid->tile_sy * level->resolution; + unitwidth = igl->grid->tile_sx * level->resolution; + + level->maxy = ceil((igl->grid->extent.maxy-igl->grid->extent.miny - 0.01* unitheight)/unitheight); + level->maxx = ceil((igl->grid->extent.maxx-igl->grid->extent.minx - 0.01* unitwidth)/unitwidth); + igl->grid->levels[i] = level; + } + igl->grid_limits = (mapcache_extent_i*)apr_pcalloc(ctx->pool,igl->grid->nlevels*sizeof(mapcache_extent_i)); + mapcache_grid_compute_limits(igl->grid,extent,igl->grid_limits,tolerance); + } APR_ARRAY_PUSH(tileset->grid_links,mapcache_grid_link*) = gridlink; } @@ -749,12 +832,6 @@ GC_CHECK_ERROR(ctx); } - if ((cur_node = ezxml_child(node,"timedimension")) != NULL) { - parseTimeDimension(ctx, cur_node, tileset); - GC_CHECK_ERROR(ctx); - } - - if ((cur_node = ezxml_child(node,"cache")) != NULL) { mapcache_cache *cache = mapcache_configuration_get_cache(config, cur_node->txt); if(!cache) { @@ -762,7 +839,7 @@ " but it is not configured", name, cur_node->txt); return; } - tileset->cache = cache; + tileset->_cache = cache; } if ((cur_node = ezxml_child(node,"source")) != NULL) { @@ -910,6 +987,7 @@ } + void mapcache_configuration_parse_xml(mapcache_context *ctx, const char *filename, mapcache_cfg *config) { ezxml_t doc, node; @@ -1039,8 +1117,6 @@ } else if ((node = ezxml_child(doc,"services")) != NULL) { ctx->log(ctx,MAPCACHE_WARN," tag is deprecated, use "); parseServices(ctx, node, config); - } else { - ctx->set_error(ctx, 400, "no configured"); } if(GC_HAS_ERROR(ctx)) goto cleanup; @@ -1078,20 +1154,12 @@ } } - if((node = ezxml_child(doc,"lock_dir")) != NULL) { - config->lockdir = apr_pstrdup(ctx->pool, node->txt); + if((node = ezxml_child(doc,"locker")) != NULL) { + mapcache_config_parse_locker(ctx,node,&config->locker); + GC_CHECK_ERROR(ctx); } else { - config->lockdir = apr_pstrdup(ctx->pool,"/tmp"); - } - - if((node = ezxml_child(doc,"lock_retry")) != NULL) { - char *endptr; - config->lock_retry_interval = (unsigned int)strtol(node->txt,&endptr,10); - if(*endptr != 0 || config->lock_retry_interval < 0) { - ctx->set_error(ctx, 400, "failed to parse lock_retry microseconds \"%s\". Expecting a positive integer", - node->txt); - return; - } + mapcache_config_parse_locker_old(ctx,doc,config); + GC_CHECK_ERROR(ctx); } if((node = ezxml_child(doc,"threaded_fetching")) != NULL) { diff -Nru mapcache-1.2.1/lib/connection_pool.c mapcache-1.6.1/lib/connection_pool.c --- mapcache-1.2.1/lib/connection_pool.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/connection_pool.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,193 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache connection pooling + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +#include +#include "mapcache.h" + +struct mapcache_connection_pool { + apr_pool_t *server_pool; + apr_reslist_t *connexions; +}; + + +struct mapcache_pooled_connection_container { + mapcache_pooled_connection *head; + apr_pool_t *pool; + unsigned int max_list_size; +}; + + + +struct mapcache_pooled_connection_private_data { + char *key; + mapcache_connection_destructor destructor; + mapcache_pooled_connection *next; + mapcache_pooled_connection_container *pcc; +}; + +static apr_status_t mapcache_connection_container_creator(void **conn_, void *params, apr_pool_t *pool) { + mapcache_pooled_connection_container *pcc; + pcc = calloc(1, sizeof(mapcache_pooled_connection_container)); + pcc->max_list_size = 10; + pcc->pool = pool; + *conn_ = pcc; + return APR_SUCCESS; +} + +static apr_status_t mapcache_connection_container_destructor(void *conn_, void *params, apr_pool_t *pool) { + mapcache_pooled_connection_container *pcc = (mapcache_pooled_connection_container*)conn_; + mapcache_pooled_connection *pc = pcc->head; + while(pc) { + mapcache_pooled_connection *this = pc; + this->private->destructor(this->connection); + free(this->private->key); + pc = this->private->next; + free(this); + } + free(pcc); + return MAPCACHE_SUCCESS; +} + + +apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool) { + apr_status_t rv; + *cp = apr_pcalloc(server_pool, sizeof(mapcache_connection_pool)); + (*cp)->server_pool = server_pool; + rv = apr_reslist_create(&((*cp)->connexions), 1, 5, 1024, 60*1000000, + mapcache_connection_container_creator, + mapcache_connection_container_destructor, + NULL, + server_pool); + return rv; +} + +mapcache_pooled_connection* mapcache_connection_pool_get_connection(mapcache_context *ctx, char *key, + mapcache_connection_constructor constructor, mapcache_connection_destructor destructor, + void *params) { + apr_status_t rv; + int count = 0; + mapcache_pooled_connection_container *pcc; + mapcache_pooled_connection *pc,*pred=NULL; + rv = apr_reslist_acquire(ctx->connection_pool->connexions, (void**)&pcc); + if(rv != APR_SUCCESS || !pcc) { + char errmsg[120]; + ctx->set_error(ctx,500, "failed to acquire connection from mapcache connection pool: (%s)", apr_strerror(rv, errmsg,120)); + return NULL; + } + + /* loop through existing connections to see if we find one matching the given key */ + pc = pcc->head; + while(pc) { + count++; + if(!strcmp(key,pc->private->key)) { + /* move current connection to head of list, and return it. We only move the connection + to the front of the list if it wasn't in the first 2 connections, as in the seeding + case we are always alternating between read and write operations (i.e. potentially + 2 different connections and in that cas we end up switching connections each time + there's an access */ + if(pc != pcc->head && count>2) { + assert(pred); + pred->private->next = pc->private->next; + pc->private->next = pcc->head; + pcc->head = pc; + } + return pc; + } + pred = pc; + pc = pc->private->next; + } + + /* connection not found in pool */ + pc = calloc(1,sizeof(mapcache_pooled_connection)); + /* + ctx->log(ctx, MAPCACHE_DEBUG, "calling constructor for pooled connection (%s)", key); + */ + constructor(ctx, &pc->connection, params); + if(GC_HAS_ERROR(ctx)) { + free(pc); + apr_reslist_release(ctx->connection_pool->connexions, pcc); + return NULL; + } + + pc->private = calloc(1,sizeof(mapcache_pooled_connection_private_data)); + pc->private->key = strdup(key); + pc->private->destructor = destructor; + pc->private->next = pcc->head; + pc->private->pcc = pcc; + + if(count == pcc->max_list_size) { + /* max number of connections atained, we must destroy the last one that was used */ + mapcache_pooled_connection *opc; + opc = pcc->head; + count = 1; + while(count < pcc->max_list_size) { + pred = opc; + opc = opc->private->next; + count++; + } + ctx->log(ctx, MAPCACHE_DEBUG, "tearing down pooled connection (%s) to make room", opc->private->key); + opc->private->destructor(opc->connection); + free(opc->private->key); + free(opc->private); + free(opc); + if(pred) { + pred->private->next = NULL; + } + } + pcc->head = pc; + return pc; + +} +void mapcache_connection_pool_invalidate_connection(mapcache_context *ctx, mapcache_pooled_connection *connection) { + mapcache_pooled_connection_container *pcc = connection->private->pcc; + mapcache_pooled_connection *pc = pcc->head, *pred=NULL; + while(pc) { + if(pc == connection) { + if(pred) { + pred->private->next = pc->private->next; + } else { + pcc->head = pc->private->next; + } + pc->private->destructor(pc->connection); + free(pc->private->key); + free(pc); + break; + } + pred = pc; + pc = pc->private->next; + } + apr_reslist_release(ctx->connection_pool->connexions,(void*)pcc); +} + +void mapcache_connection_pool_release_connection(mapcache_context *ctx, mapcache_pooled_connection *connection) { + if(connection) { + mapcache_pooled_connection_container *pcc = connection->private->pcc; + apr_reslist_release(ctx->connection_pool->connexions,(void*)pcc); + } +} + diff -Nru mapcache-1.2.1/lib/core.c mapcache-1.6.1/lib/core.c --- mapcache-1.2.1/lib/core.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/core.c 2017-09-29 21:31:13.000000000 +0000 @@ -196,10 +196,13 @@ { int expires = 0; mapcache_http_response *response; - int i,is_empty=1 /* response image is initially empty */; char *timestr; - mapcache_image *base=NULL; - mapcache_image_format *format = NULL; + mapcache_image *base; + mapcache_image_format *format; + mapcache_image_format_type t; + int i,is_empty=1; /* response image is initially empty */; + base=NULL; + format = NULL; #ifdef DEBUG if(req_tile->ntiles ==0) { @@ -209,11 +212,22 @@ #endif response = mapcache_http_response_create(ctx->pool); + if(ctx->supports_redirects && req_tile->ntiles == 1) { + req_tile->tiles[0]->allow_redirect = 1; + } + mapcache_prefetch_tiles(ctx,req_tile->tiles,req_tile->ntiles); if(GC_HAS_ERROR(ctx)) return NULL; + if(req_tile->tiles[0]->redirect) { + response->code = 302; + apr_table_set(response->headers,"Location",req_tile->tiles[0]->redirect); + response->data = mapcache_buffer_create(0, ctx->pool); + return response; + } + /* loop through tiles, and eventually merge them vertically together */ for(i=0; intiles; i++) { mapcache_tile *tile = req_tile->tiles[i]; /* shortcut */ @@ -222,7 +236,7 @@ if(tile->expires && (tile->expires < expires || expires == 0)) { expires = tile->expires; } - + if(tile->nodata) { /* treat the special case where the cache explicitely stated that the tile was empty, and we don't have any vertical merging to do */ @@ -233,8 +247,8 @@ } continue; } - - /* treat the most common case: + + /* treat the most common case: - we have a single tile request (i.e. isempty is true) - the cache returned the encoded image */ @@ -271,7 +285,7 @@ mapcache_image_merge(ctx, base, tile->raw_image); } else { /* we don't need to merge onto an existing tile and don't have access to the tile's encoded data. - * + * * we don't encode the tile's raw image data just yet because we might need to merge another one on top * of it later. */ @@ -283,8 +297,8 @@ if(!response->data) { /* we need to encode the raw image data*/ if(base) { - if(req_tile->format) { - format = req_tile->format; + if(req_tile->image_request.format) { + format = req_tile->image_request.format; } else { format = req_tile->tiles[0]->tileset->format; if(!format) { @@ -296,20 +310,20 @@ return NULL; } } else { + unsigned char empty[5] = {'#',0,0,0,0}; #ifdef DEBUG if(!is_empty) { ctx->set_error(ctx,500,"BUG: no image data to encode, but tile not marked as empty"); return NULL; } #endif - unsigned char empty[5] = {'#',0,0,0,0}; - response->data = mapcache_empty_png_decode(ctx,empty,&is_empty); /* is_empty is unchanged and left to 1 */ + response->data = mapcache_empty_png_decode(ctx,req_tile->tiles[0]->grid_link->grid->tile_sx, req_tile->tiles[0]->grid_link->grid->tile_sy, empty,&is_empty); /* is_empty is unchanged and left to 1 */ format = mapcache_configuration_get_image_format(ctx->config,"PNG8"); } } - + /* compute the content-type */ - mapcache_image_format_type t = mapcache_imageio_header_sniff(ctx,response->data); + t = mapcache_imageio_header_sniff(ctx,response->data); if(t == GC_PNG) apr_table_set(response->headers,"Content-Type","image/png"); else if(t == GC_JPEG) @@ -334,15 +348,18 @@ mapcache_tile ***maptiles; int *nmaptiles; mapcache_tile **tiles; + mapcache_grid_link **effectively_used_grid_links; mapcache_map *basemap = NULL; int ntiles = 0; int i; maptiles = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_tile**)); nmaptiles = apr_pcalloc(ctx->pool,nmaps*sizeof(int)); + effectively_used_grid_links = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_grid_link*)); for(i=0; itileset,maps[i]->grid_link, &maps[i]->extent, maps[i]->width, maps[i]->height, - &(nmaptiles[i]), &(maptiles[i])); + &(nmaptiles[i]), &(maptiles[i]), &(effectively_used_grid_links[i])); + if(GC_HAS_ERROR(ctx)) return NULL; ntiles += nmaptiles[i]; } tiles = apr_pcalloc(ctx->pool,ntiles * sizeof(mapcache_tile*)); @@ -379,7 +396,7 @@ } } if(hasdata) { - maps[i]->raw_image = mapcache_tileset_assemble_map_tiles(ctx,maps[i]->tileset,maps[i]->grid_link, + maps[i]->raw_image = mapcache_tileset_assemble_map_tiles(ctx,maps[i]->tileset,effectively_used_grid_links[i], &maps[i]->extent, maps[i]->width, maps[i]->height, nmaptiles[i], maptiles[i], mode); @@ -441,7 +458,7 @@ return NULL; } } - basemap->tileset->source->render_map(ctx, basemap); + mapcache_source_render_map(ctx, basemap->tileset->source, basemap); if(GC_HAS_ERROR(ctx)) return NULL; if(req_map->nmaps>1) { if(!basemap->raw_image) { @@ -450,7 +467,7 @@ } for(i=1; inmaps; i++) { mapcache_map *overlaymap = req_map->maps[i]; - overlaymap->tileset->source->render_map(ctx, overlaymap); + mapcache_source_render_map(ctx, overlaymap->tileset->source, overlaymap); if(GC_HAS_ERROR(ctx)) return NULL; if(!overlaymap->raw_image) { overlaymap->raw_image = mapcache_imageio_decode(ctx,overlaymap->encoded_data); @@ -468,7 +485,7 @@ } if(basemap->raw_image) { - format = req_map->getmap_format; /* always defined, defaults to JPEG */ + format = req_map->image_request.format; /* always defined, defaults to JPEG */ response->data = format->write(ctx,basemap->raw_image,format); if(GC_HAS_ERROR(ctx)) { return NULL; @@ -516,16 +533,21 @@ mapcache_http *http; mapcache_http_response *response = mapcache_http_response_create(ctx->pool); response->data = mapcache_buffer_create(30000,ctx->pool); - http = req_proxy->http; + http = mapcache_http_clone(ctx, req_proxy->rule->http); if(req_proxy->pathinfo) { - http = mapcache_http_clone(ctx,http); if( (*(req_proxy->pathinfo)) == '/' || http->url[strlen(http->url)-1] == '/') http->url = apr_pstrcat(ctx->pool,http->url,req_proxy->pathinfo,NULL); else http->url = apr_pstrcat(ctx->pool,http->url,"/",req_proxy->pathinfo,NULL); } - mapcache_http_do_request_with_params(ctx,http,req_proxy->params,response->data,response->headers,&response->code); + http->url = mapcache_http_build_url(ctx,http->url,req_proxy->params); + http->post_body = req_proxy->post_buf; + http->post_len = req_proxy->post_len; + if(req_proxy->headers) { + apr_table_overlap(http->headers, req_proxy->headers, APR_OVERLAP_TABLES_SET); + } + mapcache_http_do_request(ctx,http, response->data,response->headers,&response->code); if(response->code !=0 && GC_HAS_ERROR(ctx)) { /* the http request was successful, but the server returned an error */ ctx->clear_errors(ctx); @@ -557,7 +579,7 @@ ctx->set_error(ctx,404, "unsupported feature info format %s",fi->format); return NULL; } - tileset->source->query_info(ctx,fi); + mapcache_source_query_info(ctx, tileset->source, fi); if(GC_HAS_ERROR(ctx)) return NULL; response = mapcache_http_response_create(ctx->pool); response->data = fi->data; @@ -598,7 +620,7 @@ msg = ctx->_errmsg; if(!msg) { - msg = apr_pstrdup(ctx->pool,"an unspecified error has occured"); + msg = apr_pstrdup(ctx->pool,"an unspecified error has occurred"); } ctx->log(ctx,MAPCACHE_ERROR,msg); diff -Nru mapcache-1.2.1/lib/dimension.c mapcache-1.6.1/lib/dimension.c --- mapcache-1.2.1/lib/dimension.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/dimension.c 2017-09-29 21:31:13.000000000 +0000 @@ -35,126 +35,159 @@ #include #ifdef USE_SQLITE #include -#include -#include -#ifdef APR_HAS_THREADS -#include +#include #endif +#ifdef USE_PCRE +#include +#else +#include #endif +typedef struct mapcache_dimension_time mapcache_dimension_time; +typedef struct mapcache_dimension_time_sqlite mapcache_dimension_time_sqlite; +typedef struct mapcache_dimension_intervals mapcache_dimension_intervals; +typedef struct mapcache_dimension_values mapcache_dimension_values; +typedef struct mapcache_dimension_sqlite mapcache_dimension_sqlite; +typedef struct mapcache_dimension_regex mapcache_dimension_regex; +typedef struct mapcache_dimension_composite mapcache_dimension_composite; + +struct mapcache_dimension_values { + mapcache_dimension dimension; + apr_array_header_t *values; + int case_sensitive; +}; +struct mapcache_dimension_sqlite { + mapcache_dimension dimension; + char *dbfile; + char *get_values_for_entry_query; + char *get_all_values_query; +}; -static int _mapcache_dimension_intervals_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +struct mapcache_dimension_regex { + mapcache_dimension dimension; + char *regex_string; +#ifdef USE_PCRE + pcre *pcregex; +#else + regex_t *regex; +#endif +}; + +struct mapcache_dimension_time { + mapcache_dimension_sqlite dimension; +}; + + +#ifndef HAVE_TIMEGM +time_t timegm(struct tm *tm) { - int i; - char *endptr; - mapcache_dimension_intervals *dimension; - double val = strtod(*value,&endptr); - *value = apr_psprintf(ctx->pool,"%g",val); - if(*endptr != 0) { - return MAPCACHE_FAILURE; - } - dimension = (mapcache_dimension_intervals*)dim; - for(i=0; inintervals; i++) { - double diff; - mapcache_interval *interval = &dimension->intervals[i]; - if(valstart || val>interval->end) - continue; - if(interval->resolution == 0) - return MAPCACHE_SUCCESS; - diff = fmod((val - interval->start),interval->resolution); - if(diff == 0.0) - return MAPCACHE_SUCCESS; - } - return MAPCACHE_FAILURE; + time_t t, tdiff; + struct tm in, gtime, ltime; + + memcpy(&in, tm, sizeof(in)); + t = mktime(&in); + + memcpy(>ime, gmtime(&t), sizeof(gtime)); + memcpy(<ime, localtime(&t), sizeof(ltime)); + gtime.tm_isdst = ltime.tm_isdst; + tdiff = t - mktime(>ime); + + memcpy(&in, tm, sizeof(in)); + return mktime(&in) + tdiff; } +#endif -static const char** _mapcache_dimension_intervals_print(mapcache_context *ctx, mapcache_dimension *dim) -{ - mapcache_dimension_intervals *dimension = (mapcache_dimension_intervals*)dim; - const char **ret = (const char**)apr_pcalloc(ctx->pool,(dimension->nintervals+1)*sizeof(const char*)); - int i; - for(i=0; inintervals; i++) { - mapcache_interval *interval = &dimension->intervals[i]; - ret[i] = apr_psprintf(ctx->pool,"%g/%g/%g",interval->start,interval->end,interval->resolution); +apr_array_header_t *mapcache_requested_dimensions_clone(apr_pool_t *pool, apr_array_header_t *src) { + apr_array_header_t *ret = NULL; + if(src) { + int i; + ret = apr_array_make(pool,src->nelts,sizeof(mapcache_requested_dimension*)); + for(i=0; inelts; i++) { + mapcache_requested_dimension *tiledim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + mapcache_requested_dimension *srcdim = APR_ARRAY_IDX(src,i,mapcache_requested_dimension*); + *tiledim = *srcdim; + APR_ARRAY_PUSH(ret,mapcache_requested_dimension*) = tiledim; + } } - ret[i]=NULL; return ret; } - -static void _mapcache_dimension_intervals_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, - ezxml_t node) -{ - mapcache_dimension_intervals *dimension; - char *key,*last; - char *values; - const char *entry = node->txt; - int count = 1; - if(!entry || !*entry) { - ctx->set_error(ctx,400,"failed to parse dimension values: none supplied"); +void mapcache_set_requested_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value) { + int i; + if(!dimensions || dimensions->nelts <= 0) { + ctx->set_error(ctx,500,"BUG: no dimensions configure for tile/map"); return; } - dimension = (mapcache_dimension_intervals*)dim; - values = apr_pstrdup(ctx->pool,entry); - - for(key=values; *key; key++) if(*key == ',') count++; - - dimension->intervals = (mapcache_interval*)apr_pcalloc(ctx->pool,count*sizeof(mapcache_interval)); - - for (key = apr_strtok(values, ",", &last); key != NULL; - key = apr_strtok(NULL, ",", &last)) { - char *endptr; - mapcache_interval *interval = &dimension->intervals[dimension->nintervals]; - interval->start = strtod(key,&endptr); - if(*endptr != '/') { - ctx->set_error(ctx,400,"failed to parse min dimension value \"%s\" in \"%s\" for dimension %s",key,entry,dim->name); + for(i=0;inelts;i++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(dimensions,i,mapcache_requested_dimension*); + if(!strcasecmp(dim->dimension->name,name)) { + dim->requested_value = value?apr_pstrdup(ctx->pool,value):NULL; return; } - key = endptr+1; + } + ctx->set_error(ctx,500,"BUG: dimension (%s) not found in tile/map",name); +} - interval->end = strtod(key,&endptr); - if(*endptr != '/') { - ctx->set_error(ctx,400,"failed to parse max dimension value \"%s\" in \"%s\" for dimension %s",key,entry,dim->name); - return; - } - key = endptr+1; - interval->resolution = strtod(key,&endptr); - if(*endptr != '\0') { - ctx->set_error(ctx,400,"failed to parse resolution dimension value \"%s\" in \"%s\" for dimension %s",key,entry,dim->name); +void mapcache_set_cached_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value) { + int i; + if(!dimensions || dimensions->nelts <= 0) { + ctx->set_error(ctx,500,"BUG: no dimensions configure for tile/map"); + return; + } + for(i=0;inelts;i++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(dimensions,i,mapcache_requested_dimension*); + if(!strcasecmp(dim->dimension->name,name)) { + dim->cached_value = value?apr_pstrdup(ctx->pool,value):NULL; return; } - dimension->nintervals++; } + ctx->set_error(ctx,500,"BUG: dimension (%s) not found in tile/map",name); +} - if(!dimension->nintervals) { - ctx->set_error(ctx, 400, " \"%s\" has no intervals",dim->name); - return; - } +void mapcache_tile_set_cached_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value) { + mapcache_set_cached_dimension(ctx,tile->dimensions,name,value); } -static int _mapcache_dimension_regex_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +void mapcache_map_set_cached_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value) { + mapcache_set_cached_dimension(ctx,map->dimensions,name,value); +} +void mapcache_tile_set_requested_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value) { + mapcache_set_requested_dimension(ctx,tile->dimensions,name,value); +} + +void mapcache_map_set_requested_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value) { + mapcache_set_requested_dimension(ctx,map->dimensions,name,value); +} + +static apr_array_header_t* _mapcache_dimension_regex_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dim, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_regex *dimension = (mapcache_dimension_regex*)dim; + apr_array_header_t *values = apr_array_make(ctx->pool,1,sizeof(char*)); #ifdef USE_PCRE int ovector[30]; - int rc = pcre_exec(dimension->pcregex,NULL,*value,strlen(*value),0,0,ovector,30); - if(rc>0) - return MAPCACHE_SUCCESS; + int rc = pcre_exec(dimension->pcregex,NULL,value,strlen(value),0,0,ovector,30); + if(rc>0) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); + } #else - if(!regexec(dimension->regex,*value,0,0,0)) { - return MAPCACHE_SUCCESS; + if(!regexec(dimension->regex,value,0,0,0)) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); } #endif - return MAPCACHE_FAILURE; + else { + ctx->set_error(ctx,400,"failed to validate requested value for dimension (%s)",dim->name); + } + return values; } -static const char** _mapcache_dimension_regex_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_regex_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_regex *dimension = (mapcache_dimension_regex*)dim; - const char **ret = (const char**)apr_pcalloc(ctx->pool,2*sizeof(const char*)); - ret[0]=dimension->regex_string; - ret[1]=NULL; + apr_array_header_t *ret = apr_array_make(ctx->pool,1,sizeof(char*)); + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,dimension->regex_string); return ret; } @@ -163,32 +196,35 @@ ezxml_t node) { mapcache_dimension_regex *dimension; - const char *entry = node->txt; - if(!entry || !*entry) { - ctx->set_error(ctx,400,"failed to parse dimension regex: none supplied"); + ezxml_t child_node = ezxml_child(node,"regex"); + + dimension = (mapcache_dimension_regex*)dim; + + if(child_node && child_node->txt && *child_node->txt) { + dimension->regex_string = apr_pstrdup(ctx->pool,child_node->txt); + } else { + ctx->set_error(ctx,400,"failed to parse dimension regex: no child supplied"); return; } - dimension = (mapcache_dimension_regex*)dim; - dimension->regex_string = apr_pstrdup(ctx->pool,entry); #ifdef USE_PCRE { const char *pcre_err; int pcre_offset; - dimension->pcregex = pcre_compile(entry,0,&pcre_err, &pcre_offset,0); + dimension->pcregex = pcre_compile(dimension->regex_string,0,&pcre_err, &pcre_offset,0); if(!dimension->pcregex) { ctx->set_error(ctx,400,"failed to compile regular expression \"%s\" for dimension \"%s\": %s", - entry,dim->name,pcre_err); + dimension->regex_string,dim->name,pcre_err); return; } } #else { - int rc = regcomp(dimension->regex, entry, REG_EXTENDED); + int rc = regcomp(dimension->regex, dimension->regex_string, REG_EXTENDED); if(rc) { char errmsg[200]; regerror(rc,dimension->regex,errmsg,200); ctx->set_error(ctx,400,"failed to compile regular expression \"%s\" for dimension \"%s\": %s", - entry,dim->name,errmsg); + dimension->regex_string,dim->name,errmsg); return; } } @@ -196,31 +232,41 @@ } -static int _mapcache_dimension_values_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +static apr_array_header_t* _mapcache_dimension_values_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dim, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { int i; mapcache_dimension_values *dimension = (mapcache_dimension_values*)dim; - for(i=0; invalues; i++) { + apr_array_header_t *values = apr_array_make(ctx->pool,1,sizeof(char*)); + for(i=0; ivalues->nelts; i++) { + char *cur_val = APR_ARRAY_IDX(dimension->values,i,char*); if(dimension->case_sensitive) { - if(!strcmp(*value,dimension->values[i])) - return MAPCACHE_SUCCESS; + if(!strcmp(value,cur_val)) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); + break; + } } else { - if(!strcasecmp(*value,dimension->values[i])) - return MAPCACHE_SUCCESS; + if(!strcasecmp(value,cur_val)) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); + break; + } } } - return MAPCACHE_FAILURE; + if(i == dimension->values->nelts) { + ctx->set_error(ctx,400,"failed to validate requested value for dimension (%s)",dim->name); + } + return values; } -static const char** _mapcache_dimension_values_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_values_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_values *dimension = (mapcache_dimension_values*)dim; - const char **ret = (const char**)apr_pcalloc(ctx->pool,(dimension->nvalues+1)*sizeof(const char*)); + apr_array_header_t *ret = apr_array_make(ctx->pool,dimension->values->nelts,sizeof(char*)); int i; - for(i=0; invalues; i++) { - ret[i] = dimension->values[i]; + for(i=0; ivalues->nelts; i++) { + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,APR_ARRAY_IDX(dimension->values,i,char*)); } - ret[i]=NULL; return ret; } @@ -228,245 +274,333 @@ static void _mapcache_dimension_values_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, ezxml_t node) { - int count = 1; mapcache_dimension_values *dimension; - const char *case_sensitive; - char *key,*last; - char *values; - const char *entry = node->txt; - if(!entry || !*entry) { - ctx->set_error(ctx,400,"failed to parse dimension values: none supplied"); + ezxml_t child_node = ezxml_child(node,"value"); + dimension = (mapcache_dimension_values*)dim; + + if(!child_node) { + ctx->set_error(ctx,400,"failed to parse dimension values: no children supplied"); return; } - - dimension = (mapcache_dimension_values*)dim; - case_sensitive = ezxml_attr(node,"case_sensitive"); - if(case_sensitive && !strcasecmp(case_sensitive,"true")) { - dimension->case_sensitive = 1; + for(; child_node; child_node = child_node->next) { + const char* entry = child_node->txt; + if(!entry || !*entry) { + ctx->set_error(ctx,400,"failed to parse dimension values: empty "); + return; + } + APR_ARRAY_PUSH(dimension->values,char*) = apr_pstrdup(ctx->pool,entry); } - values = apr_pstrdup(ctx->pool,entry); - for(key=values; *key; key++) if(*key == ',') count++; - - dimension->values = (char**)apr_pcalloc(ctx->pool,count*sizeof(char*)); - - for (key = apr_strtok(values, ",", &last); key != NULL; - key = apr_strtok(NULL, ",", &last)) { - dimension->values[dimension->nvalues]=key; - dimension->nvalues++; + child_node = ezxml_child(node,"case_sensitive"); + if(child_node && child_node->txt) { + if(!strcasecmp(child_node->txt,"true")) { + dimension->case_sensitive = 1; + } } - if(!dimension->nvalues) { + + if(!dimension->values->nelts) { ctx->set_error(ctx, 400, " \"%s\" has no values",dim->name); return; } } -mapcache_dimension* mapcache_dimension_values_create(apr_pool_t *pool) +mapcache_dimension* mapcache_dimension_values_create(mapcache_context *ctx, apr_pool_t *pool) { mapcache_dimension_values *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_values)); dimension->dimension.type = MAPCACHE_DIMENSION_VALUES; - dimension->nvalues = 0; - dimension->dimension.validate = _mapcache_dimension_values_validate; + dimension->values = apr_array_make(pool,1,sizeof(char*)); + dimension->dimension.get_entries_for_value = _mapcache_dimension_values_get_entries_for_value; dimension->dimension.configuration_parse_xml = _mapcache_dimension_values_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_values_print; + dimension->dimension.get_all_entries = _mapcache_dimension_values_get_all_entries; + dimension->dimension.get_all_ogc_formatted_entries = _mapcache_dimension_values_get_all_entries; return (mapcache_dimension*)dimension; } -mapcache_dimension* mapcache_dimension_time_create(apr_pool_t *pool) -{ - mapcache_dimension_time *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_time)); - dimension->dimension.type = MAPCACHE_DIMENSION_TIME; - dimension->nintervals = 0; -// dimension->dimension.validate = _mapcache_dimension_time_validate; -// dimension->dimension.parse = _mapcache_dimension_time_parse; -// dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_time_print; - return (mapcache_dimension*)dimension; -} - -mapcache_dimension* mapcache_dimension_intervals_create(apr_pool_t *pool) -{ - mapcache_dimension_intervals *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_intervals)); - dimension->dimension.type = MAPCACHE_DIMENSION_INTERVALS; - dimension->nintervals = 0; - dimension->dimension.validate = _mapcache_dimension_intervals_validate; - dimension->dimension.configuration_parse_xml = _mapcache_dimension_intervals_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_intervals_print; - return (mapcache_dimension*)dimension; -} -mapcache_dimension* mapcache_dimension_regex_create(apr_pool_t *pool) +mapcache_dimension* mapcache_dimension_regex_create(mapcache_context *ctx, apr_pool_t *pool) { mapcache_dimension_regex *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_regex)); dimension->dimension.type = MAPCACHE_DIMENSION_REGEX; #ifndef USE_PCRE dimension->regex = (regex_t*)apr_pcalloc(pool, sizeof(regex_t)); #endif - dimension->dimension.validate = _mapcache_dimension_regex_validate; + dimension->dimension.get_entries_for_value = _mapcache_dimension_regex_get_entries_for_value; dimension->dimension.configuration_parse_xml = _mapcache_dimension_regex_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_regex_print; + dimension->dimension.get_all_entries = _mapcache_dimension_regex_get_all_entries; + dimension->dimension.get_all_ogc_formatted_entries = _mapcache_dimension_regex_get_all_entries; return (mapcache_dimension*)dimension; } #ifdef USE_SQLITE -static apr_hash_t *time_connection_pools = NULL; - -struct sqlite_time_conn { +struct sqlite_dimension_conn { sqlite3 *handle; - sqlite3_stmt *prepared_statement; - char *errmsg; + sqlite3_stmt **prepared_statements; + int n_statements; }; -static apr_status_t _sqlite_time_reslist_get_ro_connection(void **conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_dimension_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; - int flags; - mapcache_timedimension_sqlite *dim = (mapcache_timedimension_sqlite*) params; - struct sqlite_time_conn *conn = apr_pcalloc(pool, sizeof (struct sqlite_time_conn)); + int flags; + char *dbfile = (char*) params; + struct sqlite_dimension_conn *conn = calloc(1, sizeof (struct sqlite_dimension_conn)); *conn_ = conn; flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; - ret = sqlite3_open_v2(dim->dbfile, &conn->handle, flags, NULL); - + ret = sqlite3_open_v2(dbfile, &conn->handle, flags, NULL); + if (ret != SQLITE_OK) { - return APR_EGENERAL; + ctx->set_error(ctx,500,"failed to open sqlite dimension dbfile (%s): %s",dbfile,sqlite3_errmsg(conn->handle)); + sqlite3_close(conn->handle); + *conn_=NULL; + return; } sqlite3_busy_timeout(conn->handle, 300000); - return APR_SUCCESS; } -static apr_status_t _sqlite_time_reslist_free_connection(void *conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_dimension_connection_destructor(void *conn_) { - struct sqlite_time_conn *conn = (struct sqlite_time_conn*) conn_; - if(conn->prepared_statement) { - sqlite3_finalize(conn->prepared_statement); + struct sqlite_dimension_conn *conn = (struct sqlite_dimension_conn*) conn_; + while(conn->n_statements) { + conn->n_statements--; + if(conn->prepared_statements[conn->n_statements]) { + sqlite3_finalize(conn->prepared_statements[conn->n_statements]); + } } + free(conn->prepared_statements); sqlite3_close(conn->handle); - return APR_SUCCESS; + free(conn); } -static struct sqlite_time_conn* _sqlite_time_get_conn(mapcache_context *ctx, mapcache_timedimension_sqlite *dim) { - apr_status_t rv; - struct sqlite_time_conn *conn = NULL; - apr_reslist_t *pool = NULL; - if(!time_connection_pools || NULL == (pool = apr_hash_get(time_connection_pools,dim->timedimension.key, APR_HASH_KEY_STRING)) ) { - -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - - if(!time_connection_pools) { - time_connection_pools = apr_hash_make(ctx->process_pool); - } - - /* probably doesn't exist, unless the previous mutex locked us, so we check */ - pool = apr_hash_get(time_connection_pools,dim->timedimension.key, APR_HASH_KEY_STRING); - if(!pool) { - /* there where no existing connection pools, create them*/ - rv = apr_reslist_create(&pool, - 0 /* min */, - 10 /* soft max */, - 200 /* hard max */, - 60*1000000 /*60 seconds, ttl*/, - _sqlite_time_reslist_get_ro_connection, /* resource constructor */ - _sqlite_time_reslist_free_connection, /* resource destructor */ - dim, ctx->process_pool); - if(rv != APR_SUCCESS) { - ctx->set_error(ctx,500,"failed to create sqlite time connection pool"); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(time_connection_pools,dim->timedimension.key,APR_HASH_KEY_STRING,pool); - } -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - pool = apr_hash_get(time_connection_pools,dim->timedimension.key, APR_HASH_KEY_STRING); - assert(pool); - } - rv = apr_reslist_acquire(pool, (void **) &conn); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to aquire connection to time dimension sqlite backend: %s", (conn && conn->errmsg)?conn->errmsg:"unknown error"); - return NULL; - } - return conn; +static mapcache_pooled_connection* _sqlite_dimension_get_conn(mapcache_context *ctx, mapcache_tileset *tileset, mapcache_dimension_sqlite *dim) { + mapcache_dimension *pdim = (mapcache_dimension*)dim; + char *conn_key = apr_pstrcat(ctx->pool,"dim_",tileset?tileset->name:"","_",pdim->name,NULL); + mapcache_pooled_connection *pc = mapcache_connection_pool_get_connection(ctx,conn_key, + mapcache_sqlite_dimension_connection_constructor, + mapcache_sqlite_dimension_connection_destructor, dim->dbfile); + return pc; } -static void _sqlite_time_release_conn(mapcache_context *ctx, mapcache_timedimension_sqlite *sdim, struct sqlite_time_conn *conn) +static void _sqlite_dimension_release_conn(mapcache_context *ctx, mapcache_pooled_connection *pc) { - apr_reslist_t *pool; - pool = apr_hash_get(time_connection_pools,sdim->timedimension.key, APR_HASH_KEY_STRING); - - if (GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(pool, (void*) conn); + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); } else { - apr_reslist_release(pool, (void*) conn); + mapcache_connection_pool_release_connection(ctx,pc); } } -static void _bind_sqlite_timedimension_params(mapcache_context *ctx, sqlite3_stmt *stmt, - sqlite3 *handle, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, - time_t start, time_t end) -{ +static void _mapcache_dimension_sqlite_bind_parameters(mapcache_context *ctx, sqlite3_stmt *stmt, sqlite3 *handle, + const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { int paramidx,ret; - paramidx = sqlite3_bind_parameter_index(stmt, ":tileset"); + paramidx = sqlite3_bind_parameter_index(stmt, ":dim"); if (paramidx) { - ret = sqlite3_bind_text(stmt, paramidx, tileset->name, -1, SQLITE_STATIC); + ret = sqlite3_bind_text(stmt, paramidx, value, -1, SQLITE_STATIC); if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :tileset: %s", sqlite3_errmsg(handle)); + ctx->set_error(ctx,400, "sqlite dimension failed to bind :dim : %s", sqlite3_errmsg(handle)); return; } } - if(grid) { - paramidx = sqlite3_bind_parameter_index(stmt, ":gridsrs"); + if(tileset) { + paramidx = sqlite3_bind_parameter_index(stmt, ":tileset"); if (paramidx) { - ret = sqlite3_bind_text(stmt, paramidx, grid->srs, -1, SQLITE_STATIC); + ret = sqlite3_bind_text(stmt, paramidx, tileset->name, -1, SQLITE_STATIC); if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :gridsrs %s", sqlite3_errmsg(handle)); + ctx->set_error(ctx,400, "sqlite dimension failed to bind :tileset : %s", sqlite3_errmsg(handle)); return; } } } - if(extent) { - paramidx = sqlite3_bind_parameter_index(stmt, ":minx"); + if(grid) { + paramidx = sqlite3_bind_parameter_index(stmt, ":gridsrs"); if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->minx); + ret = sqlite3_bind_text(stmt, paramidx, grid->srs, -1, SQLITE_STATIC); if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :minx %s", sqlite3_errmsg(handle)); + ctx->set_error(ctx,400, "failed to bind :gridsrs %s", sqlite3_errmsg(handle)); return; } } - paramidx = sqlite3_bind_parameter_index(stmt, ":miny"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->miny); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :miny %s", sqlite3_errmsg(handle)); - return; - } + } + + paramidx = sqlite3_bind_parameter_index(stmt, ":minx"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->minx:-DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :minx %s", sqlite3_errmsg(handle)); + return; } - paramidx = sqlite3_bind_parameter_index(stmt, ":maxx"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->maxx); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :maxx %s", sqlite3_errmsg(handle)); - return; - } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":miny"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->miny:-DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :miny %s", sqlite3_errmsg(handle)); + return; } - paramidx = sqlite3_bind_parameter_index(stmt, ":maxy"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent->maxy); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :maxy %s", sqlite3_errmsg(handle)); - return; - } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":maxx"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxx:DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :maxx %s", sqlite3_errmsg(handle)); + return; + } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":maxy"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxy:DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :maxy %s", sqlite3_errmsg(handle)); + return; + } + } +} +static apr_array_header_t* _mapcache_dimension_sqlite_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dim, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) +{ + mapcache_dimension_sqlite *dimension = (mapcache_dimension_sqlite*)dim; + struct sqlite_dimension_conn *conn = NULL; + mapcache_pooled_connection *pc; + apr_array_header_t *values = apr_array_make(ctx->pool,1,sizeof(char*)); + int sqliteret; + pc = _sqlite_dimension_get_conn(ctx,tileset,dimension); + if (GC_HAS_ERROR(ctx)) { + return values; + } + conn = pc->connection; + if(!conn->prepared_statements) { + conn->prepared_statements = calloc(2,sizeof(sqlite3_stmt*)); + conn->n_statements = 2; + } + if(!conn->prepared_statements[0]) { + if(SQLITE_OK != sqlite3_prepare_v2(conn->handle, dimension->get_values_for_entry_query, -1, &conn->prepared_statements[0], NULL)) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); + goto cleanup; } } + _mapcache_dimension_sqlite_bind_parameters(ctx,conn->prepared_statements[0],conn->handle,value, + tileset,extent,grid); + if (GC_HAS_ERROR(ctx)) { + return values; + } + + + do { + sqliteret = sqlite3_step(conn->prepared_statements[0]); + if (sqliteret != SQLITE_DONE && sqliteret != SQLITE_ROW && sqliteret != SQLITE_BUSY && sqliteret != SQLITE_LOCKED) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on query : %s (%d)", sqlite3_errmsg(conn->handle), sqliteret); + goto cleanup; + } + if(sqliteret == SQLITE_ROW) { + const char* dimrow = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,dimrow); + } + } while (sqliteret == SQLITE_ROW || sqliteret == SQLITE_BUSY || sqliteret == SQLITE_LOCKED); + +cleanup: + if(conn->prepared_statements[0]) { + sqlite3_reset(conn->prepared_statements[0]); + } + _sqlite_dimension_release_conn(ctx,pc); + + return values; +} + +static apr_array_header_t* _mapcache_dimension_sqlite_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) +{ + mapcache_dimension_sqlite *dimension = (mapcache_dimension_sqlite*)dim; + struct sqlite_dimension_conn *conn = NULL; + int sqliteret; + apr_array_header_t *ret = apr_array_make(ctx->pool,0,sizeof(char*)); + mapcache_pooled_connection *pc; + pc = _sqlite_dimension_get_conn(ctx,tileset,dimension); + if (GC_HAS_ERROR(ctx)) { + return ret; + } + conn = pc->connection; + if(!conn->prepared_statements) { + conn->prepared_statements = calloc(2,sizeof(sqlite3_stmt*)); + conn->n_statements = 2; + } + + if(!conn->prepared_statements[1]) { + sqliteret = sqlite3_prepare_v2(conn->handle, dimension->get_all_values_query, -1, &conn->prepared_statements[1], NULL); + if(sqliteret != SQLITE_OK) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); + goto cleanup; + } + } + _mapcache_dimension_sqlite_bind_parameters(ctx,conn->prepared_statements[1],conn->handle,NULL,tileset,extent,grid); + if (GC_HAS_ERROR(ctx)) { + return ret; + } + do { + sqliteret = sqlite3_step(conn->prepared_statements[1]); + if (sqliteret != SQLITE_DONE && sqliteret != SQLITE_ROW && sqliteret != SQLITE_BUSY && sqliteret != SQLITE_LOCKED) { + ctx->set_error(ctx, 500, "sqlite dimension backend failed on query : %s (%d)", sqlite3_errmsg(conn->handle), sqliteret); + goto cleanup; + } + if(sqliteret == SQLITE_ROW) { + const char* sqdim = (const char*) sqlite3_column_text(conn->prepared_statements[1], 0); + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,sqdim); + } + } while (sqliteret == SQLITE_ROW || sqliteret == SQLITE_BUSY || sqliteret == SQLITE_LOCKED); + +cleanup: + if(conn->prepared_statements[1]) { + sqlite3_reset(conn->prepared_statements[1]); + } + _sqlite_dimension_release_conn(ctx,pc); + + return ret; +} + + +static void _mapcache_dimension_sqlite_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, + ezxml_t node) +{ + mapcache_dimension_sqlite *dimension; + ezxml_t child; + + dimension = (mapcache_dimension_sqlite*)dim; + + child = ezxml_child(node,"dbfile"); + if(child) { + dimension->dbfile = apr_pstrdup(ctx->pool, child->txt); + } else { + ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); + return; + } + child = ezxml_child(node,"validate_query"); + if(child) { + dimension->get_values_for_entry_query = apr_pstrdup(ctx->pool, child->txt); + } else { + ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); + return; + } + child = ezxml_child(node,"list_query"); + if(child) { + dimension->get_all_values_query = apr_pstrdup(ctx->pool, child->txt); + } else { + ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); + return; + } + +} + + +static void _bind_sqlite_dimension_time_params(mapcache_context *ctx, sqlite3_stmt *stmt, + sqlite3 *handle, const char *dim_value, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, + time_t start, time_t end) +{ + int paramidx,ret; + + _mapcache_dimension_sqlite_bind_parameters(ctx,stmt,handle,dim_value,tileset,extent,grid); + paramidx = sqlite3_bind_parameter_index(stmt, ":start_timestamp"); if (paramidx) { ret = sqlite3_bind_int64(stmt, paramidx, start); @@ -486,57 +620,63 @@ } } -apr_array_header_t *_mapcache_timedimension_sqlite_get_entries(mapcache_context *ctx, mapcache_timedimension *dim, - mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end) { - mapcache_timedimension_sqlite *sdim = (mapcache_timedimension_sqlite*)dim; - int ret; - sqlite3_stmt *stmt; +apr_array_header_t *_mapcache_dimension_time_get_entries(mapcache_context *ctx, mapcache_dimension_time *dim, const char *dim_value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid, time_t *intervals, int n_intervals) { + mapcache_dimension_sqlite *sdim = (mapcache_dimension_sqlite*)dim; + int i,ret; apr_array_header_t *time_ids = NULL; - struct sqlite_time_conn *conn = _sqlite_time_get_conn(ctx, sdim); + mapcache_pooled_connection *pc; + struct sqlite_dimension_conn *conn; + pc = _sqlite_dimension_get_conn(ctx,tileset,sdim); if (GC_HAS_ERROR(ctx)) { - if(conn) _sqlite_time_release_conn(ctx, sdim, conn); return NULL; } - stmt = conn->prepared_statement; - if(!stmt) { - ret = sqlite3_prepare_v2(conn->handle, sdim->query, -1, &conn->prepared_statement, NULL); + conn = pc->connection; + if(!conn->prepared_statements) { + conn->prepared_statements = calloc(1,sizeof(sqlite3_stmt*)); + conn->n_statements = 1; + } + if(!conn->prepared_statements[0]) { + ret = sqlite3_prepare_v2(conn->handle, sdim->get_values_for_entry_query, -1, &conn->prepared_statements[0], NULL); if(ret != SQLITE_OK) { ctx->set_error(ctx, 500, "time sqlite backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); - _sqlite_time_release_conn(ctx, sdim, conn); + _sqlite_dimension_release_conn(ctx, pc); return NULL; } - stmt = conn->prepared_statement; } - _bind_sqlite_timedimension_params(ctx,stmt,conn->handle,tileset,grid,extent,start,end); - if(GC_HAS_ERROR(ctx)) { - sqlite3_reset(stmt); - _sqlite_time_release_conn(ctx, sdim, conn); - return NULL; + for(i=0;iprepared_statements[0],conn->handle,dim_value,tileset,grid,extent,intervals[i*2],intervals[i*2+1]); + if(GC_HAS_ERROR(ctx)) { + _sqlite_dimension_release_conn(ctx, pc); + return NULL; + } + + time_ids = apr_array_make(ctx->pool,0,sizeof(char*)); + do { + ret = sqlite3_step(conn->prepared_statements[0]); + if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { + ctx->set_error(ctx, 500, "sqlite backend failed on dimension_time query : %s (%d)", sqlite3_errmsg(conn->handle), ret); + _sqlite_dimension_release_conn(ctx, pc); + return NULL; + } + if(ret == SQLITE_ROW) { + const char* time_id = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); + APR_ARRAY_PUSH(time_ids,char*) = apr_pstrdup(ctx->pool,time_id); + } + } while (ret == SQLITE_ROW || ret == SQLITE_BUSY || ret == SQLITE_LOCKED); + sqlite3_reset(conn->prepared_statements[0]); } - - time_ids = apr_array_make(ctx->pool,0,sizeof(char*)); - do { - ret = sqlite3_step(stmt); - if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { - ctx->set_error(ctx, 500, "sqlite backend failed on timedimension query : %s (%d)", sqlite3_errmsg(conn->handle), ret); - sqlite3_reset(stmt); - _sqlite_time_release_conn(ctx, sdim, conn); - return NULL; - } - if(ret == SQLITE_ROW) { - const char* time_id = (const char*) sqlite3_column_text(stmt, 0); - APR_ARRAY_PUSH(time_ids,char*) = apr_pstrdup(ctx->pool,time_id); - } - } while (ret == SQLITE_ROW || ret == SQLITE_BUSY || ret == SQLITE_LOCKED); - sqlite3_reset(stmt); - _sqlite_time_release_conn(ctx, sdim, conn); + _sqlite_dimension_release_conn(ctx, pc); return time_ids; } -apr_array_header_t *_mapcache_timedimension_sqlite_get_all_entries(mapcache_context *ctx, mapcache_timedimension *dim, - mapcache_tileset *tileset) { - return _mapcache_timedimension_sqlite_get_entries(ctx,dim,tileset,NULL,NULL,0,INT_MAX); + +apr_array_header_t* _mapcache_dimension_time_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { + mapcache_dimension_time *tdim = (mapcache_dimension_time*)dim; + time_t all[2] = {0,INT_MAX}; + return _mapcache_dimension_time_get_entries(ctx,tdim,NULL,tileset,extent,grid,all,1); } #endif @@ -551,30 +691,31 @@ #ifdef USE_SQLITE -void _mapcache_timedimension_sqlite_parse_xml(mapcache_context *ctx, mapcache_timedimension *dim, ezxml_t node) { - mapcache_timedimension_sqlite *sdim = (mapcache_timedimension_sqlite*)dim; +void _mapcache_dimension_time_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, ezxml_t node) { + mapcache_dimension_sqlite *sdim = (mapcache_dimension_sqlite*)dim; + mapcache_dimension *pdim = (mapcache_dimension*)dim; ezxml_t child; - + child = ezxml_child(node,"dbfile"); if(child && child->txt && *child->txt) { sdim->dbfile = apr_pstrdup(ctx->pool,child->txt); } else { - ctx->set_error(ctx,400,"no entry for %s",dim->key); + ctx->set_error(ctx,400,"no entry for %s",pdim->name); return; } child = ezxml_child(node,"query"); if(child && child->txt && *child->txt) { - sdim->query = apr_pstrdup(ctx->pool,child->txt); + sdim->get_values_for_entry_query = apr_pstrdup(ctx->pool,child->txt); } else { - ctx->set_error(ctx,400,"no entry for %s",dim->key); + ctx->set_error(ctx,400,"no entry for %s",pdim->name); return; } } #endif char *mapcache_ogc_strptime(const char *value, struct tm *ts, mapcache_time_interval_t *ti) { - memset (ts, '\0', sizeof (*ts)); char *valueptr; + memset (ts, '\0', sizeof (*ts)); valueptr = strptime(value,"%Y-%m-%dT%H:%M:%SZ",ts); *ti = MAPCACHE_TINTERVAL_SECOND; if(valueptr) return valueptr; @@ -596,76 +737,122 @@ return NULL; } -apr_array_header_t* mapcache_timedimension_get_entries_for_value(mapcache_context *ctx, mapcache_timedimension *timedimension, - mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, const char *value) { +#ifdef USE_SQLITE +apr_array_header_t* _mapcache_dimension_time_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dimension, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { + /* look if supplied value is a predefined key */ /* split multiple values, loop */ /* extract start and end values */ struct tm tm_start,tm_end; - time_t start,end; + time_t *intervals; mapcache_time_interval_t tis,tie; - char *valueptr = (char*)value; - valueptr = mapcache_ogc_strptime(value,&tm_start,&tis); - if(!valueptr) { - ctx->set_error(ctx,400,"failed to parse time %s",value); - return NULL; - } + char *valueptr = apr_pstrdup(ctx->pool,value); + char *last,*key; + int count=0; + mapcache_dimension_time *dimension_time = (mapcache_dimension_time*)dimension; + + /*count how many time entries were supplied*/ + for(; *value; value++) if(*value == ',') count++; - if(*valueptr == '/') { - /* we have a second (end) time */ - valueptr++; - valueptr = mapcache_ogc_strptime(valueptr,&tm_end,&tie); + intervals = apr_pcalloc(ctx->pool,2*count*sizeof(time_t)); + count = 0; + + + /* Split the input on '&' */ + for (key = apr_strtok(valueptr, ",", &last); key != NULL; + key = apr_strtok(NULL, ",", &last)) { + valueptr = mapcache_ogc_strptime(key,&tm_start,&tis); if(!valueptr) { - ctx->set_error(ctx,400,"failed to parse end time in %s",value); + ctx->set_error(ctx,400,"failed to parse time %s",value); return NULL; } - } else if(*valueptr == 0) { - tie = tis; - tm_end = tm_start; - } else { - ctx->set_error(ctx,400,"failed (2) to parse time %s",value); - return NULL; - } - end = timegm(&tm_end); - start = timegm(&tm_start); - if(difftime(start,end) == 0) { - switch(tie) { - case MAPCACHE_TINTERVAL_SECOND: - tm_end.tm_sec += 1; - break; - case MAPCACHE_TINTERVAL_MINUTE: - tm_end.tm_min += 1; - break; - case MAPCACHE_TINTERVAL_HOUR: - tm_end.tm_hour += 1; - break; - case MAPCACHE_TINTERVAL_DAY: - tm_end.tm_mday += 1; - break; - case MAPCACHE_TINTERVAL_MONTH: - tm_end.tm_mon += 1; - break; - case MAPCACHE_TINTERVAL_YEAR: - tm_end.tm_year += 1; - break; + + if(*valueptr == '/' || (*valueptr == '-' && *(valueptr+1) == '-')) { + /* we have a second (end) time */ + if (*valueptr == '/') { + valueptr++; + } + else { + valueptr += 2; + } + valueptr = mapcache_ogc_strptime(valueptr,&tm_end,&tie); + if(!valueptr) { + ctx->set_error(ctx,400,"failed to parse end time in %s",value); + return NULL; + } + } else if(*valueptr == 0) { + tie = tis; + tm_end = tm_start; + } else { + ctx->set_error(ctx,400,"failed (2) to parse time %s",value); + return NULL; + } + intervals[count*2+1] = timegm(&tm_end); + intervals[count*2] = timegm(&tm_start); + if(difftime(intervals[count*2],intervals[count*2+1]) == 0) { + switch(tie) { + case MAPCACHE_TINTERVAL_SECOND: + tm_end.tm_sec += 1; + break; + case MAPCACHE_TINTERVAL_MINUTE: + tm_end.tm_min += 1; + break; + case MAPCACHE_TINTERVAL_HOUR: + tm_end.tm_hour += 1; + break; + case MAPCACHE_TINTERVAL_DAY: + tm_end.tm_mday += 1; + break; + case MAPCACHE_TINTERVAL_MONTH: + tm_end.tm_mon += 1; + break; + case MAPCACHE_TINTERVAL_YEAR: + tm_end.tm_year += 1; + break; + } + intervals[count*2+1] = timegm(&tm_end); } - end = timegm(&tm_end); + count++; } - - return timedimension->get_entries_for_interval(ctx,timedimension,tileset,grid,extent,start,end); + return _mapcache_dimension_time_get_entries(ctx,dimension_time,value,tileset,extent,grid,intervals,count); /* end loop */ } +#endif +mapcache_dimension* mapcache_dimension_sqlite_create(mapcache_context *ctx, apr_pool_t *pool) +{ #ifdef USE_SQLITE -mapcache_timedimension* mapcache_timedimension_sqlite_create(apr_pool_t *pool) { - mapcache_timedimension_sqlite *dim = apr_pcalloc(pool, sizeof(mapcache_timedimension_sqlite)); - dim->timedimension.assembly_type = MAPCACHE_TIMEDIMENSION_ASSEMBLY_STACK; - dim->timedimension.get_entries_for_interval = _mapcache_timedimension_sqlite_get_entries; - dim->timedimension.get_all_entries = _mapcache_timedimension_sqlite_get_all_entries; - dim->timedimension.configuration_parse_xml = _mapcache_timedimension_sqlite_parse_xml; - return (mapcache_timedimension*)dim; + mapcache_dimension_sqlite *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_sqlite)); + dimension->dimension.type = MAPCACHE_DIMENSION_SQLITE; + dimension->dbfile = NULL; + dimension->dimension.get_entries_for_value = _mapcache_dimension_sqlite_get_entries_for_value; + dimension->dimension.configuration_parse_xml = _mapcache_dimension_sqlite_parse_xml; + dimension->dimension.get_all_entries = _mapcache_dimension_sqlite_get_all_entries; + dimension->dimension.get_all_ogc_formatted_entries = _mapcache_dimension_sqlite_get_all_entries; + return (mapcache_dimension*)dimension; +#else + ctx->set_error(ctx,400,"Sqlite dimension support requires SQLITE support to be built in"); + return NULL; +#endif } + + +mapcache_dimension* mapcache_dimension_time_create(mapcache_context *ctx, apr_pool_t *pool) { +#ifdef USE_SQLITE + mapcache_dimension_time *dim = apr_pcalloc(pool, sizeof(mapcache_dimension_time)); + mapcache_dimension *pdim = (mapcache_dimension*)dim; + pdim->get_entries_for_value = _mapcache_dimension_time_get_entries_for_value; + pdim->get_all_entries = _mapcache_dimension_time_get_all_entries; + pdim->get_all_ogc_formatted_entries = _mapcache_dimension_time_get_all_entries; + pdim->configuration_parse_xml = _mapcache_dimension_time_parse_xml; + return pdim; +#else + ctx->set_error(ctx,400,"TIME dimension support requires SQLITE support to be built in"); + return NULL; #endif +} + /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/lib/ezxml.c mapcache-1.6.1/lib/ezxml.c --- mapcache-1.2.1/lib/ezxml.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/ezxml.c 2017-09-29 21:31:13.000000000 +0000 @@ -370,7 +370,7 @@ else *s = '\0'; // null terminate tag name for (i = 0; root->attr[i] && strcmp(n, root->attr[i][0]); i++); - while (*(n = ++s + strspn(s, EZXML_WS)) && *n != '>') { + while (*(n = s + 1 + strspn(s + 1, EZXML_WS)) && *n != '>') { if (*(s = n + strcspn(n, EZXML_WS))) *s = '\0'; // attr name else { ezxml_err(root, t, "malformed levels[z]->resolution; @@ -71,6 +71,65 @@ } } +void mapcache_grid_get_metatile_extent(mapcache_context *ctx, mapcache_tile *tile, mapcache_extent *extent) { + mapcache_grid *grid = tile->grid_link->grid; + double res = grid->levels[tile->z]->resolution; + double gbuffer,gwidth,gheight,fullgwidth,fullgheight; + mapcache_tileset *tileset = tile->tileset; + int mtx,mty,blx,bly,mtsx,mtsy; + mtx = tile->x / tileset->metasize_x; + if(tile->x < 0) + mtx --; + mty = tile->y / tileset->metasize_y; + if(tile->y < 0) + mty --; + blx = mtx * tileset->metasize_x; + bly = mty * tileset->metasize_y; + + /* adjust the size of the the metatile so it does not extend past the grid limits. + * If we don't do this, we end up with cut labels on the edges of the tile grid + */ + if(blx+tileset->metasize_x-1 >= grid->levels[tile->z]->maxx) { + mtsx = grid->levels[tile->z]->maxx - blx; + } else { + mtsx = tileset->metasize_x; + } + if(bly+tileset->metasize_y-1 >= grid->levels[tile->z]->maxy) { + mtsy = grid->levels[tile->z]->maxy - bly; + } else { + mtsy = tileset->metasize_y; + } + + /* buffer in geographical units */ + gbuffer = res * tileset->metabuffer; + + /* adjusted metatile size in geographical units */ + gwidth = res * mtsx * grid->tile_sx; + gheight = res * mtsy * grid->tile_sy; + + /* configured metatile size in geographical units */ + fullgwidth = res * tileset->metasize_x * grid->tile_sx; + fullgheight = res * tileset->metasize_y * grid->tile_sy; + + switch(grid->origin) { + case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: + extent->minx = grid->extent.minx + mtx * fullgwidth - gbuffer; + extent->miny = grid->extent.miny + mty * fullgheight - gbuffer; + extent->maxx = extent->minx + gwidth + 2 * gbuffer; + extent->maxy = extent->miny + gheight + 2 * gbuffer; + break; + case MAPCACHE_GRID_ORIGIN_TOP_LEFT: + extent->minx = grid->extent.minx + mtx * fullgwidth - gbuffer; + extent->maxy = grid->extent.maxy - mty * fullgheight + gbuffer; + extent->maxx = extent->minx + gwidth + 2 * gbuffer; + extent->miny = extent->maxy - gheight - 2 * gbuffer; + break; + case MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT: + case MAPCACHE_GRID_ORIGIN_TOP_RIGHT: + ctx->set_error(ctx,500,"origin not implemented"); + } +} + const char* mapcache_grid_get_crs(mapcache_context *ctx, mapcache_grid *grid) { char *epsgnum; @@ -160,10 +219,11 @@ return MAPCACHE_FAILURE; } -void mapcache_grid_get_closest_level(mapcache_context *ctx, mapcache_grid_link *grid_link, double resolution, int *level) +mapcache_grid_link* mapcache_grid_get_closest_wms_level(mapcache_context *ctx, mapcache_grid_link *grid_link, double resolution, int *level) { double dst = fabs(grid_link->grid->levels[grid_link->minz]->resolution - resolution); - int i; + int i,g; + mapcache_grid_link *ret = grid_link; *level = 0; for(i=grid_link->minz + 1; imaxz; i++) { @@ -173,6 +233,20 @@ *level = i; } } + if(grid_link->intermediate_grids) { + for(g=0; gintermediate_grids->nelts; g++) { + mapcache_grid_link *igl = APR_ARRAY_IDX(grid_link->intermediate_grids, g, mapcache_grid_link*); + for(i=igl->minz; imaxz; i++) { + double curdst = fabs(igl->grid->levels[i]->resolution - resolution); + if(curdst + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include "mapcache.h" +#include +#include + +#define SHA256_DIGEST_SIZE ( 256 / 8) +#define SHA256_BLOCK_SIZE ( 512 / 8) + +typedef unsigned char uint8; +typedef unsigned int uint32; +typedef unsigned long long uint64; + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA256_BLOCK_SIZE]; + uint32 h[8]; +} sha256_ctx; + +typedef struct { + sha256_ctx ctx_inside; + sha256_ctx ctx_outside; + + /* for hmac_reinit */ + sha256_ctx ctx_inside_reinit; + sha256_ctx ctx_outside_reinit; + + unsigned char block_ipad[SHA256_BLOCK_SIZE]; + unsigned char block_opad[SHA256_BLOCK_SIZE]; +} hmac_sha256_ctx; + + +#define SHFR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) +#define CH(x, y, z) ((x & y) ^ (~x & z)) +#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) + +#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3)) +#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10)) + +#define UNPACK32(x, str) \ +{ \ + *((str) + 3) = (uint8) ((x) ); \ + *((str) + 2) = (uint8) ((x) >> 8); \ + *((str) + 1) = (uint8) ((x) >> 16); \ + *((str) + 0) = (uint8) ((x) >> 24); \ +} + +#define PACK32(str, x) \ +{ \ + *(x) = ((uint32) *((str) + 3) ) \ + | ((uint32) *((str) + 2) << 8) \ + | ((uint32) *((str) + 1) << 16) \ + | ((uint32) *((str) + 0) << 24); \ +} + +#define UNPACK64(x, str) \ +{ \ + *((str) + 7) = (uint8) ((x) ); \ + *((str) + 6) = (uint8) ((x) >> 8); \ + *((str) + 5) = (uint8) ((x) >> 16); \ + *((str) + 4) = (uint8) ((x) >> 24); \ + *((str) + 3) = (uint8) ((x) >> 32); \ + *((str) + 2) = (uint8) ((x) >> 40); \ + *((str) + 1) = (uint8) ((x) >> 48); \ + *((str) + 0) = (uint8) ((x) >> 56); \ +} + +#define PACK64(str, x) \ +{ \ + *(x) = ((uint64) *((str) + 7) ) \ + | ((uint64) *((str) + 6) << 8) \ + | ((uint64) *((str) + 5) << 16) \ + | ((uint64) *((str) + 4) << 24) \ + | ((uint64) *((str) + 3) << 32) \ + | ((uint64) *((str) + 2) << 40) \ + | ((uint64) *((str) + 1) << 48) \ + | ((uint64) *((str) + 0) << 56); \ +} + +#define SHA256_SCR(i) \ +{ \ + w[i] = SHA256_F4(w[i - 2]) + w[i - 7] \ + + SHA256_F3(w[i - 15]) + w[i - 16]; \ +} + +uint32 sha256_h0[8] = + {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + +uint32 sha256_k[64] = + {0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +void sha256_transf(sha256_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint32 w[64]; + uint32 wv[8]; + uint32 t1, t2; + const unsigned char *sub_block; + int i,j; + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 6); + + for (j = 0; j < 16; j++) { + PACK32(&sub_block[j << 2], &w[j]); + } + + for (j = 16; j < 64; j++) { + SHA256_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } + } +} + + +void sha256_init(sha256_ctx *ctx) +{ + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha256_h0[i]; + } + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA256_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA256_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA256_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA256_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha256_final(sha256_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + int i; + + block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) + < (ctx->len % SHA256_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + + for (i = 0 ; i < 8; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +} + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest) +{ + sha256_ctx ctx; + + sha256_init(&ctx); + sha256_update(&ctx, message, len); + sha256_final(&ctx, digest); +} + +void hmac_sha256_init(hmac_sha256_ctx *ctx, const unsigned char *key, + unsigned int key_size) +{ + unsigned int fill; + unsigned int num; + + const unsigned char *key_used; + unsigned char key_temp[SHA256_DIGEST_SIZE]; + int i; + + if (key_size == SHA256_BLOCK_SIZE) { + key_used = key; + num = SHA256_BLOCK_SIZE; + } else { + if (key_size > SHA256_BLOCK_SIZE){ + num = SHA256_DIGEST_SIZE; + sha256(key, key_size, key_temp); + key_used = key_temp; + } else { /* key_size > SHA256_BLOCK_SIZE */ + key_used = key; + num = key_size; + } + fill = SHA256_BLOCK_SIZE - num; + + memset(ctx->block_ipad + num, 0x36, fill); + memset(ctx->block_opad + num, 0x5c, fill); + } + + for (i = 0; i < (int) num; i++) { + ctx->block_ipad[i] = key_used[i] ^ 0x36; + ctx->block_opad[i] = key_used[i] ^ 0x5c; + } + + sha256_init(&ctx->ctx_inside); + sha256_update(&ctx->ctx_inside, ctx->block_ipad, SHA256_BLOCK_SIZE); + + sha256_init(&ctx->ctx_outside); + sha256_update(&ctx->ctx_outside, ctx->block_opad, + SHA256_BLOCK_SIZE); + + /* for hmac_reinit */ + memcpy(&ctx->ctx_inside_reinit, &ctx->ctx_inside, + sizeof(sha256_ctx)); + memcpy(&ctx->ctx_outside_reinit, &ctx->ctx_outside, + sizeof(sha256_ctx)); +} + +void hmac_sha256_reinit(hmac_sha256_ctx *ctx) +{ + memcpy(&ctx->ctx_inside, &ctx->ctx_inside_reinit, + sizeof(sha256_ctx)); + memcpy(&ctx->ctx_outside, &ctx->ctx_outside_reinit, + sizeof(sha256_ctx)); +} + +void hmac_sha256_update(hmac_sha256_ctx *ctx, const unsigned char *message, + unsigned int message_len) +{ + sha256_update(&ctx->ctx_inside, message, message_len); +} + +void hmac_sha256_final(hmac_sha256_ctx *ctx, unsigned char *mac, + unsigned int mac_size) +{ + unsigned char digest_inside[SHA256_DIGEST_SIZE]; + unsigned char mac_temp[SHA256_DIGEST_SIZE]; + + sha256_final(&ctx->ctx_inside, digest_inside); + sha256_update(&ctx->ctx_outside, digest_inside, SHA256_DIGEST_SIZE); + sha256_final(&ctx->ctx_outside, mac_temp); + memcpy(mac, mac_temp, mac_size); +} + +void hmac_sha256(const unsigned char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + unsigned char *mac, unsigned mac_size) +{ + hmac_sha256_ctx ctx; + + hmac_sha256_init(&ctx, key, key_size); + hmac_sha256_update(&ctx, message, message_len); + hmac_sha256_final(&ctx, mac, mac_size); +} + +void sha_hex_encode(unsigned char *sha, unsigned int sha_size) { + int i = sha_size; + while(i--) { + char hex[3]; + sprintf(hex, "%02x", sha[i]); + memcpy(sha+2*i,hex,2); + } +} + + +#define HMAC_IPAD 0x36 +#define HMAC_OPAD 0x5c +#define HMAC_BLOCKSIZE 64 +#include + +static void mxor(void *dst, const void *src, size_t len) { + char const *s = src; + char *d = dst; + for (; len > 0; len--) + *d++ ^= *s++; + return; +} + +void hmac_sha1(const char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + void *mac) { + + apr_sha1_ctx_t inner; + apr_sha1_ctx_t outer; + unsigned char keypad[HMAC_BLOCKSIZE]; + unsigned char inner_digest[APR_SHA1_DIGESTSIZE]; + unsigned char long_digest[APR_SHA1_DIGESTSIZE]; + + /* Shorten the key down to the blocksize, anything more is useless */ + if (key_size > HMAC_BLOCKSIZE) { + apr_sha1_ctx_t context; + apr_sha1_init(&context); + apr_sha1_update_binary(&context, key, key_size); + apr_sha1_final(long_digest, &context); + key = long_digest; + key_size = APR_SHA1_DIGESTSIZE; + } + + /* Prepare and mask the inner portion of the key */ + memset(keypad, HMAC_IPAD, HMAC_BLOCKSIZE); + mxor(keypad, key, key_size); + + /* Compute the inner hash */ + apr_sha1_init(&inner); + apr_sha1_update_binary(&inner, keypad, HMAC_BLOCKSIZE); + apr_sha1_update(&inner, message, message_len); + apr_sha1_final(inner_digest, &inner); + + /* Prepare and mask the outer portion of the key */ + memset(keypad, HMAC_OPAD, HMAC_BLOCKSIZE); + mxor(keypad, key, key_size); + + /* Compute the outer hash */ + apr_sha1_init(&outer); + apr_sha1_update_binary(&outer, keypad, HMAC_BLOCKSIZE); + apr_sha1_update_binary(&outer, inner_digest, APR_SHA1_DIGESTSIZE); + apr_sha1_final(mac, &outer); +} + diff -Nru mapcache-1.2.1/lib/http.c mapcache-1.6.1/lib/http.c --- mapcache-1.2.1/lib/http.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/http.c 2017-09-29 21:31:13.000000000 +0000 @@ -73,12 +73,39 @@ return size*nmemb; } +/*update val replacing {foo_header} with the Value of foo_header from the orginal request */ +static void _header_replace_str(mapcache_context *ctx, apr_table_t *headers, char **val) { + char *value = *val; + char *start_tag, *end_tag; + size_t start_tag_offset; + start_tag = strchr(value,'{'); + while(start_tag) { + start_tag_offset = start_tag - value; /*record where we found the '{' so we can look for the next one after that spot + (avoids infinite loop if tag was not found/replaced) */ + *start_tag=0; + end_tag = strchr(start_tag+1,'}'); + if(end_tag) { + const char *header_value; + *end_tag=0; + header_value = apr_table_get(headers,start_tag+1); + if(header_value) { + value = apr_pstrcat(ctx->pool,value,header_value,end_tag+1,NULL); + } + *end_tag='}'; + } + *start_tag='{'; + start_tag = strchr(value+start_tag_offset+1,'{'); + } + *val = value; +} + void mapcache_http_do_request(mapcache_context *ctx, mapcache_http *req, mapcache_buffer *data, apr_table_t *headers, long *http_code) { CURL *curl_handle; char error_msg[CURL_ERROR_SIZE]; int ret; struct curl_slist *curl_headers=NULL; + struct _header_struct h; curl_handle = curl_easy_init(); @@ -95,7 +122,6 @@ if(headers != NULL) { /* intercept headers */ - struct _header_struct h; h.headers = headers; h.ctx=ctx; curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, _mapcache_curl_header_callback); @@ -115,19 +141,29 @@ apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; int i; for (i = 0; i < array->nelts; i++) { - curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",elts[i].val,NULL)); + char *val = elts[i].val; + if(val && strchr(val,'{') && ctx->headers_in) { + _header_replace_str(ctx,ctx->headers_in,&val); + } + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",val,NULL)); } } if(!req->headers || !apr_table_get(req->headers,"User-Agent")) { curl_headers = curl_slist_append(curl_headers, "User-Agent: "MAPCACHE_USERAGENT); } curl_easy_setopt(curl_handle, CURLOPT_HTTPHEADER, curl_headers); + + if(req->post_body && req->post_len>0) { + curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, req->post_body); + } + + if(!http_code) + curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1); + /* get it! */ ret = curl_easy_perform(curl_handle); if(http_code) curl_easy_getinfo (curl_handle, CURLINFO_RESPONSE_CODE, http_code); - else - curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1); if(ret != CURLE_OK) { ctx->set_error(ctx, 502, "curl failed to request url %s : %s", req->url, error_msg); @@ -144,16 +180,31 @@ mapcache_http_do_request(ctx,request,data,headers, http_code); } -/* calculate the length of the string formed by key=value&, and add it to cnt */ -#ifdef _WIN32 -static int _mapcache_key_value_strlen_callback(void *cnt, const char *key, const char *value) -{ -#else -static APR_DECLARE_NONSTD(int) _mapcache_key_value_strlen_callback(void *cnt, const char *key, const char *value) -{ -#endif - *((int*)cnt) += strlen(key) + 2 + ((value && *value) ? strlen(value) : 0); - return 1; +typedef struct header_cb_struct{ + apr_pool_t *pool; + char *str; +} header_cb_struct; + +/* Converts an integer value to its hex character*/ +char to_hex(char code) { + static char hex[] = "0123456789abcdef"; + return hex[code & 15]; +} + +/* Returns a url-encoded version of str */ +char *url_encode(apr_pool_t *p, const char *str) { + char *buf = apr_pcalloc(p, strlen(str) * 3 + 1), *pbuf = buf; + while (*str) { + if (isalnum(*str) || *str == '-' || *str == '_' || *str == '.' || *str == '~') + *pbuf++ = *str; + else if (*str == ' ') + *pbuf++ = '+'; + else + *pbuf++ = '%', *pbuf++ = to_hex(*str >> 4), *pbuf++ = to_hex(*str & 15); + str++; + } + *pbuf = '\0'; + return buf; } #ifdef _WIN32 @@ -163,13 +214,15 @@ static APR_DECLARE_NONSTD(int) _mapcache_key_value_append_callback(void *cnt, const char *key, const char *value) { #endif -#define _mystr *((char**)cnt) - _mystr = apr_cpystrn(_mystr,key,MAX_STRING_LEN); - *((_mystr)++) = '='; +#define _mystr (((header_cb_struct*)cnt)->str) + header_cb_struct *hcs = (header_cb_struct*)cnt; + hcs->str = apr_pstrcat(hcs->pool, hcs->str, key, "=", NULL); if(value && *value) { - _mystr = apr_cpystrn(_mystr,value,MAX_STRING_LEN); + hcs->str = apr_pstrcat(hcs->pool, hcs->str, url_encode(hcs->pool, value), "&", NULL); + } + else { + hcs->str = apr_pstrcat(hcs->pool, hcs->str, "&", NULL); } - *((_mystr)++) = '&'; return 1; #undef _mystr } @@ -230,38 +283,29 @@ -char* mapcache_http_build_url(mapcache_context *r, char *base, apr_table_t *params) +char* mapcache_http_build_url(mapcache_context *ctx, char *base, apr_table_t *params) { if(!apr_is_empty_table(params)) { - int stringLength = 0, baseLength; - char *builtUrl,*builtUrlPtr; - char charToAppend=0; + int baseLength; + header_cb_struct hcs; baseLength = strlen(base); - - /*calculate the length of the param string we are going to build */ - apr_table_do(_mapcache_key_value_strlen_callback, (void*)&stringLength, params, NULL); + hcs.pool = ctx->pool; + hcs.str = base; if(strchr(base,'?')) { /* base already contains a '?' , shall we be adding a '&' to the end */ if(base[baseLength-1] != '?' && base[baseLength-1] != '&') { - charToAppend = '&'; + hcs.str = apr_pstrcat(ctx->pool, hcs.str, "&", NULL); } } else { /* base does not contain a '?', we will be adding it */ - charToAppend='?'; + hcs.str = apr_pstrcat(ctx->pool, hcs.str, "?", NULL); } - /* add final \0 and eventual separator to add ('?' or '&') */ - stringLength += baseLength + ((charToAppend)?2:1); - - builtUrl = builtUrlPtr = apr_palloc(r->pool, stringLength); - - builtUrlPtr = apr_cpystrn(builtUrlPtr,base,MAX_STRING_LEN); - if(charToAppend) - *(builtUrlPtr++)=charToAppend; - apr_table_do(_mapcache_key_value_append_callback, (void*)&builtUrlPtr, params, NULL); - *(builtUrlPtr-1) = '\0'; /*replace final '&' by a \0 */ - return builtUrl; + apr_table_do(_mapcache_key_value_append_callback, (void*)&hcs, params, NULL); + baseLength = strlen(hcs.str); + hcs.str[baseLength-1] = '\0'; + return hcs.str; } else { return base; } diff -Nru mapcache-1.2.1/lib/image.c mapcache-1.6.1/lib/image.c --- mapcache-1.2.1/lib/image.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/image.c 2017-09-29 21:31:13.000000000 +0000 @@ -56,7 +56,7 @@ return img; } -int mapcache_image_has_alpha(mapcache_image *img) +int mapcache_image_has_alpha(mapcache_image *img, unsigned int cutoff) { size_t i,j; if(img->has_alpha == MC_ALPHA_UNKNOWN) { @@ -64,7 +64,7 @@ for(i=0; ih; i++) { ptr = rptr; for(j=0; jw; j++) { - if(ptr[3]<(unsigned char)255) { + if(ptr[3]<(unsigned char)cutoff) { img->has_alpha = MC_ALPHA_YES; return 1; } @@ -85,7 +85,11 @@ void mapcache_image_merge(mapcache_context *ctx, mapcache_image *base, mapcache_image *overlay) { int starti,startj; -#ifndef USE_PIXMAN +#ifdef USE_PIXMAN + pixman_image_t *si; + pixman_image_t *bi; + pixman_transform_t transform; +#else int i,j; unsigned char *browptr, *orowptr, *bptr, *optr; #endif @@ -94,20 +98,22 @@ ctx->set_error(ctx, 500, "attempting to merge an larger image onto another"); return; } + starti = (base->h - overlay->h)/2; startj = (base->w - overlay->w)/2; #ifdef USE_PIXMAN - pixman_image_t *si = pixman_image_create_bits(PIXMAN_a8r8g8b8,overlay->w,overlay->h, + si = pixman_image_create_bits(PIXMAN_a8r8g8b8,overlay->w,overlay->h, (uint32_t*)overlay->data,overlay->stride); - pixman_image_t *bi = pixman_image_create_bits(PIXMAN_a8r8g8b8,base->w,base->h, + bi = pixman_image_create_bits(PIXMAN_a8r8g8b8,base->w,base->h, (uint32_t*)base->data,base->stride); - pixman_transform_t transform; - pixman_transform_init_translate(&transform, - pixman_int_to_fixed(-startj), - pixman_int_to_fixed(-starti)); pixman_image_set_filter(si,PIXMAN_FILTER_NEAREST, NULL, 0); - pixman_image_set_transform (si, &transform); - pixman_image_composite (PIXMAN_OP_OVER, si, si, bi, + if(starti || startj) { + pixman_transform_init_translate(&transform, + pixman_int_to_fixed(-startj), + pixman_int_to_fixed(-starti)); + pixman_image_set_transform (si, &transform); + } + pixman_image_composite (PIXMAN_OP_OVER, si, NULL, bi, 0, 0, 0, 0, 0, 0, base->w,base->h); pixman_image_unref(si); pixman_image_unref(bi); @@ -289,6 +295,12 @@ ctx->set_error(ctx, 500, "failed to load image data from metatile"); return; } + if(metatile->w != mt->map.width || + metatile->h != mt->map.height) { + ctx->set_error(ctx, 500, "image size does not correspond to metatile size"); + return; + } + for(i=0; imetasize_x; i++) { for(j=0; jmetasize_y; j++) { tileimg = mapcache_image_create(ctx); @@ -312,6 +324,9 @@ sx = mt->map.tileset->metabuffer + i * tileimg->w; sy = mt->map.height - (mt->map.tileset->metabuffer + (j+1) * tileimg->h); break; + default: + ctx->set_error(ctx,500,"BUG: unknown grid origin"); + return; } tileimg->data = &(metatile->data[sy*metatile->stride + 4 * sx]); if(mt->map.tileset->watermark) { @@ -359,5 +374,24 @@ return MAPCACHE_FALSE; } + +void mapcache_image_fill(mapcache_context *ctx, mapcache_image *image, const unsigned char *fill_color) { +#if 0 && defined(USE_PIXMAN) + pixman_fill((uint32_t*)image->data, image->stride, 32, 0, 0, image->w, image->h, *((int*)fill_color) ); +#else + int r,c; + unsigned char *pixptr; + for(r=0;rh;r++) { + pixptr = image->data + image->stride * r; + for(c=0;cw;c++) { + pixptr[0]=fill_color[0]; + pixptr[1]=fill_color[1]; + pixptr[2]=fill_color[2]; + pixptr[3]=fill_color[3]; + pixptr+=4; + } + } +#endif +} /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/lib/imageio_jpeg.c mapcache-1.6.1/lib/imageio_jpeg.c --- mapcache-1.2.1/lib/imageio_jpeg.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/imageio_jpeg.c 2017-09-29 21:31:13.000000000 +0000 @@ -188,6 +188,18 @@ default: jpeg_set_colorspace(&cinfo, JCS_YCbCr); } + switch(((mapcache_image_format_jpeg*)format)->optimize) { + case MAPCACHE_OPTIMIZE_NO: + cinfo.optimize_coding = FALSE; + break; + case MAPCACHE_OPTIMIZE_ARITHMETIC: + cinfo.optimize_coding = FALSE; + cinfo.arith_code = TRUE; + break; + case MAPCACHE_OPTIMIZE_YES: + default: + cinfo.optimize_coding = TRUE; + } jpeg_start_compress(&cinfo, TRUE); rowdata = (JSAMPLE*)malloc(img->w*cinfo.input_components*sizeof(JSAMPLE)); @@ -317,7 +329,7 @@ } mapcache_image_format* mapcache_imageio_create_jpeg_format(apr_pool_t *pool, char *name, int quality, - mapcache_photometric photometric) + mapcache_photometric photometric, mapcache_optimization optimize) { mapcache_image_format_jpeg *format = apr_pcalloc(pool, sizeof(mapcache_image_format_jpeg)); format->format.name = name; @@ -327,6 +339,7 @@ format->format.create_empty_image = _mapcache_imageio_jpg_create_empty; format->format.write = _mapcache_imageio_jpeg_encode; format->quality = quality; + format->optimize = optimize; format->photometric = photometric; format->format.type = GC_JPEG; return (mapcache_image_format*)format; diff -Nru mapcache-1.2.1/lib/imageio_mixed.c mapcache-1.6.1/lib/imageio_mixed.c --- mapcache-1.2.1/lib/imageio_mixed.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/imageio_mixed.c 2017-09-29 21:31:13.000000000 +0000 @@ -46,7 +46,7 @@ mapcache_image *image, mapcache_image_format *format) { mapcache_image_format_mixed *f = (mapcache_image_format_mixed*)format; - if(mapcache_image_has_alpha(image)) { + if(mapcache_image_has_alpha(image,f->alpha_cutoff)) { return f->transparent->write(ctx,image,f->transparent); } else { return f->opaque->write(ctx,image,f->opaque); @@ -54,12 +54,14 @@ } mapcache_image_format* mapcache_imageio_create_mixed_format(apr_pool_t *pool, - char *name, mapcache_image_format *transparent, mapcache_image_format *opaque) + char *name, mapcache_image_format *transparent, mapcache_image_format *opaque, + unsigned int alpha_cutoff) { mapcache_image_format_mixed *format = apr_pcalloc(pool, sizeof(mapcache_image_format_mixed)); format->format.name = name; format->transparent = transparent; format->opaque = opaque; + format->alpha_cutoff = alpha_cutoff; format->format.extension = apr_pstrdup(pool,"xxx"); format->format.mime_type = NULL; format->format.write = _mapcache_imageio_mixed_encode; diff -Nru mapcache-1.2.1/lib/imageio_png.c mapcache-1.6.1/lib/imageio_png.c --- mapcache-1.2.1/lib/imageio_png.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/imageio_png.c 2017-09-29 21:31:13.000000000 +0000 @@ -45,6 +45,9 @@ #ifndef Z_BEST_COMPRESSION #define Z_BEST_COMPRESSION 9 #endif +#ifndef Z_NO_COMPRESSION +#define Z_NO_COMPRESSION 0 +#endif /* Table of CRCs of all 8-bit messages. */ #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ @@ -146,48 +149,152 @@ return update_crc(0xffffffffL, buf, len) ^ 0xffffffffL; } -static unsigned char empty_png[] = { +static unsigned char empty_png_256[] = { 0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a,0x00,0x00,0x00,0x0d,0x49,0x48,0x44,0x52 ,0x00,0x00,0x01,0x00,0x00,0x00,0x01,0x00,0x01,0x03,0x00,0x00,0x00,0x66,0xbc,0x3a - ,0x25,0x00,0x00,0x00,0x03,0x50,0x4c,0x54,0x45,0x73,0x91,0xad,0x31,0xf0,0x8f,0xdd - ,0x00,0x00,0x00,0x01,0x74,0x52,0x4e,0x53,0xff,0x6d,0xe4,0x37,0xeb,0x00,0x00,0x00 + ,0x25,0x00,0x00,0x00,0x03, + /*PLTE*/0x50,0x4c,0x54,0x45, + /*RGB*/0x73,0x91,0xad, + /*PLTE-CRC*/0x31,0xf0,0x8f,0xdd + ,0x00,0x00,0x00,0x01, + /*tRNS*/0x74,0x52,0x4e,0x53, + /*A (fully opaque by default)*/0xff, + /*tRNS-CRC*/0x6d,0xe4,0x37,0xeb, + 0x00,0x00,0x00 ,0x1f,0x49,0x44,0x41,0x54,0x68,0xde,0xed,0xc1,0x01,0x0d,0x00,0x00,0x00,0xc2,0xa0 ,0xf7,0x4f,0x6d,0x0e,0x37,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xbe,0x0d ,0x21,0x00,0x00,0x01,0x7f,0x19,0x9c,0xa7,0x00,0x00,0x00,0x00,0x49,0x45,0x4e,0x44 ,0xae,0x42,0x60,0x82 }; +static size_t plte_offset_256= 0x25; +static size_t trns_offset_256 = 0x34; + + +unsigned char empty_png_384[] = { + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, + 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x01, 0x80, + 0x01, 0x03, 0x00, 0x00, 0x00, 0x9e, 0x09, 0x27, 0xfc, 0x00, 0x00, 0x00, + 0x03, + /*PLTE*/0x50,0x4c,0x54,0x45, + /*RGB*/0xcd,0x44,0x44, + /*PLTE-CRC*/0xdb,0xa9,0x37,0x41 + ,0x00,0x00,0x00,0x01, + /*tRNS*/0x74,0x52,0x4e,0x53, + /*A (fully opaque by default)*/0xff, + /*tRNS-CRC*/0x6d,0xe4,0x37,0xeb, + 0x00,0x00,0x00, + 0x28, 0x49, 0x44, 0x41, 0x54, 0x78, 0x5e, 0xed, + 0xc1, 0x01, 0x01, 0x00, 0x00, 0x00, 0x82, 0x20, 0xff, 0xaf, 0x6e, 0x48, + 0x40, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x19, 0x49, 0x80, 0x00, + 0x01, 0x28, 0x77, 0xfb, 0x61, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, + 0x44, 0xae, 0x42, 0x60, 0x82 +}; +static size_t plte_offset_384= 0x25; +static size_t trns_offset_384 = 0x34; + + +static unsigned char empty_png_512[] = { + 0x89,0x50,0x4e,0x47,0x0d,0x0a,0x1a,0x0a,0x00,0x00,0x00,0x0d,0x49,0x48,0x44,0x52 + ,0x00,0x00,0x02,0x00,0x00,0x00,0x02,0x00,0x01,0x03,0x00,0x00,0x00,0xce,0xb6,0x46 + ,0xb9,0x00,0x00,0x00,0x03, + /*PLTE*/0x50,0x4c,0x54,0x45, + /*RGB*/0xcd,0x44,0x44, + /*PLTE-CRC*/0xdb,0xa9,0x37,0x41 + ,0x00,0x00,0x00,0x01, + /*tRNS*/0x74,0x52,0x4e,0x53, + /*A (fully opaque by default)*/0xff, + /*tRNS-CRC*/0x6d,0xe4,0x37,0xeb, + 0x00,0x00,0x00 + ,0x36,0x49,0x44,0x41,0x54,0x78,0x5e,0xed,0xc1,0x01,0x01,0x00,0x00,0x00,0x82,0x20 + ,0xff,0xaf,0x6e,0x48,0x40,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 + ,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00 + ,0x00,0x00,0x00,0x00,0x00,0x7c,0x1b,0x82,0x00,0x00,0x01,0x9a,0x38,0x6a,0xc7,0x00 + ,0x00,0x00,0x00,0x49,0x45,0x4e,0x44,0xae,0x42,0x60,0x82 +}; +static size_t plte_offset_512= 0x25; +static size_t trns_offset_512 = 0x34; -static size_t plte_offset = 0x25; -static size_t trns_offset = 0x34; -mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, const unsigned char *hex_color, int *is_empty) { +mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, int width, int height, const unsigned char *hex_color, int *is_empty) { int chunkcrc; unsigned char *dd; - mapcache_buffer *encoded_data = mapcache_buffer_create(sizeof(empty_png)+4,ctx->pool); - dd = encoded_data->buf; - memcpy(dd,empty_png,sizeof(empty_png)); - dd[plte_offset+4] = hex_color[3]; //r - dd[plte_offset+5] = hex_color[2]; //g - dd[plte_offset+6] = hex_color[1]; //b - chunkcrc = crc(dd+plte_offset,7); - dd[plte_offset+7] = (unsigned char)((chunkcrc >> 24) & 0xff); - dd[plte_offset+8] = (unsigned char)((chunkcrc >> 16) & 0xff); - dd[plte_offset+9] = (unsigned char)((chunkcrc >> 8) & 0xff); - dd[plte_offset+10] = (unsigned char)(chunkcrc & 0xff); - if(hex_color[4] != 255) { - dd[trns_offset+4] = hex_color[4]; - chunkcrc = crc(dd+trns_offset,5); - dd[trns_offset+5] = (unsigned char)((chunkcrc >> 24) & 0xff); - dd[trns_offset+6] = (unsigned char)((chunkcrc >> 16) & 0xff); - dd[trns_offset+7] = (unsigned char)((chunkcrc >> 8) & 0xff); - dd[trns_offset+8] = (unsigned char)(chunkcrc & 0xff); + mapcache_buffer *encoded_data; + if(width == 256 && height == 256) { + encoded_data = mapcache_buffer_create(sizeof(empty_png_256)+4,ctx->pool); + dd = encoded_data->buf; + memcpy(dd,empty_png_256,sizeof(empty_png_256)); + dd[plte_offset_256+4] = hex_color[3]; //r + dd[plte_offset_256+5] = hex_color[2]; //g + dd[plte_offset_256+6] = hex_color[1]; //b + chunkcrc = crc(dd+plte_offset_256,7); + dd[plte_offset_256+7] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[plte_offset_256+8] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[plte_offset_256+9] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[plte_offset_256+10] = (unsigned char)(chunkcrc & 0xff); + if(hex_color[4] != 255) { + dd[trns_offset_256+4] = hex_color[4]; + chunkcrc = crc(dd+trns_offset_256,5); + dd[trns_offset_256+5] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[trns_offset_256+6] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[trns_offset_256+7] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[trns_offset_256+8] = (unsigned char)(chunkcrc & 0xff); + } + encoded_data->size = sizeof(empty_png_256); + } else if(width == 384 && height == 384) { + encoded_data = mapcache_buffer_create(sizeof(empty_png_384)+4,ctx->pool); + dd = encoded_data->buf; + memcpy(dd,empty_png_384,sizeof(empty_png_384)); + dd[plte_offset_384+4] = hex_color[3]; //r + dd[plte_offset_384+5] = hex_color[2]; //g + dd[plte_offset_384+6] = hex_color[1]; //b + chunkcrc = crc(dd+plte_offset_384,7); + dd[plte_offset_384+7] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[plte_offset_384+8] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[plte_offset_384+9] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[plte_offset_384+10] = (unsigned char)(chunkcrc & 0xff); + if(hex_color[4] != 255) { + dd[trns_offset_384+4] = hex_color[4]; + chunkcrc = crc(dd+trns_offset_384,5); + dd[trns_offset_384+5] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[trns_offset_384+6] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[trns_offset_384+7] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[trns_offset_384+8] = (unsigned char)(chunkcrc & 0xff); + } + encoded_data->size = sizeof(empty_png_384); + } else if(width == 512 && height == 512) { + encoded_data = mapcache_buffer_create(sizeof(empty_png_512)+4,ctx->pool); + dd = encoded_data->buf; + memcpy(dd,empty_png_512,sizeof(empty_png_512)); + dd[plte_offset_512+4] = hex_color[3]; //r + dd[plte_offset_512+5] = hex_color[2]; //g + dd[plte_offset_512+6] = hex_color[1]; //b + chunkcrc = crc(dd+plte_offset_512,7); + dd[plte_offset_512+7] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[plte_offset_512+8] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[plte_offset_512+9] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[plte_offset_512+10] = (unsigned char)(chunkcrc & 0xff); + if(hex_color[4] != 255) { + dd[trns_offset_512+4] = hex_color[4]; + chunkcrc = crc(dd+trns_offset_512,5); + dd[trns_offset_512+5] = (unsigned char)((chunkcrc >> 24) & 0xff); + dd[trns_offset_512+6] = (unsigned char)((chunkcrc >> 16) & 0xff); + dd[trns_offset_512+7] = (unsigned char)((chunkcrc >> 8) & 0xff); + dd[trns_offset_512+8] = (unsigned char)(chunkcrc & 0xff); + } + encoded_data->size = sizeof(empty_png_512); + } else { + /* here for compatibility reasons, although this should not be used in production as it is cpu-heavy */ + mapcache_image *rgba = mapcache_image_create_with_data(ctx,width,height); + mapcache_image_format *format = mapcache_configuration_get_image_format(ctx->config,"PNG8"); + mapcache_image_fill(ctx,rgba,hex_color+1); + encoded_data = format->write(ctx,rgba,format); } if(hex_color[4] == 0) { *is_empty = 1; } else { *is_empty = 0; } - encoded_data->size = sizeof(empty_png); return encoded_data; } @@ -217,14 +324,10 @@ // do nothing } -#ifndef _WIN32 -static inline int premultiply (int color,int alpha) -#else -static __inline int premultiply (int color,int alpha) -#endif -{ - int temp = (alpha * color) + 0x80; - return ((temp + (temp >> 8)) >> 8); +#define PREMULTIPLY(out,color,alpha)\ +{\ + int temp = ((alpha) * (color)) + 0x80;\ + out =((temp + (temp >> 8)) >> 8);\ } @@ -318,9 +421,9 @@ pixptr[1] = 0; pixptr[2] = 0; } else { - pixptr[0] = premultiply(pixel[2],alpha); - pixptr[1] = premultiply(pixel[1],alpha); - pixptr[2] = premultiply(pixel[0],alpha); + PREMULTIPLY(pixptr[0],pixel[2],alpha); + PREMULTIPLY(pixptr[1],pixel[1],alpha); + PREMULTIPLY(pixptr[2],pixel[0],alpha); } pixptr += 4; } @@ -410,6 +513,9 @@ png_set_compression_level (png_ptr, Z_BEST_COMPRESSION); else if(compression == MAPCACHE_COMPRESSION_FAST) png_set_compression_level (png_ptr, Z_BEST_SPEED); + else if(compression == MAPCACHE_COMPRESSION_DISABLE) + png_set_compression_level (png_ptr, Z_NO_COMPRESSION); + png_set_filter(png_ptr,0,PNG_FILTER_NONE); info_ptr = png_create_info_struct(png_ptr); @@ -430,7 +536,7 @@ png_set_write_fn(png_ptr, buffer, _mapcache_imageio_png_write_func, _mapcache_imageio_png_flush_func); - if(mapcache_image_has_alpha(img)) + if(mapcache_image_has_alpha(img,255)) color_type = PNG_COLOR_TYPE_RGB_ALPHA; else color_type = PNG_COLOR_TYPE_RGB; @@ -1314,6 +1420,8 @@ png_set_compression_level (png_ptr, Z_BEST_COMPRESSION); else if(compression == MAPCACHE_COMPRESSION_FAST) png_set_compression_level (png_ptr, Z_BEST_SPEED); + else if(compression == MAPCACHE_COMPRESSION_DISABLE) + png_set_compression_level (png_ptr, Z_NO_COMPRESSION); png_set_filter(png_ptr,0,PNG_FILTER_NONE); info_ptr = png_create_info_struct(png_ptr); if (!info_ptr) { @@ -1413,7 +1521,7 @@ mapcache_image_format_png_q *format = apr_pcalloc(pool, sizeof(mapcache_image_format_png_q)); format->format.format.name = name; format->format.format.extension = apr_pstrdup(pool,"png"); - format->format.format.mime_type = apr_pstrdup(pool,"image/png"); + format->format.format.mime_type = apr_pstrdup(pool,"image/png; mode=8bit"); format->format.compression_level = compression; format->format.format.write = _mapcache_imageio_png_q_encode; format->format.format.create_empty_image = _mapcache_imageio_png_create_empty; diff -Nru mapcache-1.2.1/lib/lock.c mapcache-1.6.1/lib/lock.c --- mapcache-1.2.1/lib/lock.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/lock.c 2017-09-29 21:31:13.000000000 +0000 @@ -31,8 +31,31 @@ #include #include #include +#ifndef _WIN32 +#include +#endif + +typedef struct { + mapcache_locker locker; + + /** + * directory where lock files will be placed. + * Must be readable and writable by the apache user. + * Must be placed on a network mounted shared directory if multiple mapcache instances + * need to be synchronized + */ + const char *dir; +} mapcache_locker_disk; + +typedef struct { + mapcache_locker locker; + apr_array_header_t *lockers; +} mapcache_locker_fallback; -char* lock_filename_for_resource(mapcache_context *ctx, const char *resource) + +#define MAPCACHE_LOCKFILE_PREFIX "_gc_lock" + +char* lock_filename_for_resource(mapcache_context *ctx, mapcache_locker_disk *ldisk, const char *resource) { char *saferes = apr_pstrdup(ctx->pool,resource); char *safeptr = saferes; @@ -43,45 +66,418 @@ safeptr++; } return apr_psprintf(ctx->pool,"%s/"MAPCACHE_LOCKFILE_PREFIX"%s.lck", - ctx->config->lockdir,saferes); + ldisk->dir,saferes); } -int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, char *resource) +int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void **lock) { - char *lockname = lock_filename_for_resource(ctx,resource); + mapcache_lock_result rv = locker->aquire_lock(ctx, locker, resource, lock); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_FAILURE; + } + if(rv == MAPCACHE_LOCK_AQUIRED) + return MAPCACHE_TRUE; + else { + apr_time_t start_wait = apr_time_now(); + rv = MAPCACHE_LOCK_LOCKED; + + while(rv != MAPCACHE_LOCK_NOENT) { + unsigned int waited = apr_time_as_msec(apr_time_now()-start_wait); + if(waited > locker->timeout*1000) { + mapcache_unlock_resource(ctx,locker,*lock); + ctx->log(ctx,MAPCACHE_ERROR,"deleting a possibly stale lock after waiting on it for %g seconds",waited/1000.0); + return MAPCACHE_FALSE; + } + apr_sleep(locker->retry_interval * 1000000); + rv = locker->ping_lock(ctx,locker, *lock); + } + return MAPCACHE_FALSE; + } +} + + +mapcache_lock_result mapcache_locker_disk_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { + char *lockname, errmsg[120]; + mapcache_locker_disk *ldisk; apr_file_t *lockfile; apr_status_t rv; + + assert(self->type == MAPCACHE_LOCKER_DISK); + ldisk = (mapcache_locker_disk*)self; + + lockname = lock_filename_for_resource(ctx,ldisk,resource); + *lock = lockname; + /* create the lockfile */ rv = apr_file_open(&lockfile,lockname,APR_WRITE|APR_CREATE|APR_EXCL|APR_XTHREAD,APR_OS_DEFAULT,ctx->pool); /* if the file already exists, wait for it to disappear */ /* TODO: check the lock isn't stale (i.e. too old) */ if( rv != APR_SUCCESS ) { - apr_finfo_t info; - rv = apr_stat(&info,lockname,0,ctx->pool); -#ifdef DEBUG - if(!APR_STATUS_IS_ENOENT(rv)) { - ctx->log(ctx, MAPCACHE_DEBUG, "waiting on resource lock %s", resource); - } -#endif - while(!APR_STATUS_IS_ENOENT(rv)) { - /* sleep for the configured number of micro-seconds (default is 1/100th of a second) */ - apr_sleep(ctx->config->lock_retry_interval); - rv = apr_stat(&info,lockname,0,ctx->pool); + if( !APR_STATUS_IS_EEXIST(rv) ) { + ctx->set_error(ctx, 500, "failed to create lockfile %s: %s", lockname, apr_strerror(rv,errmsg,120)); + return MAPCACHE_LOCK_NOENT; } - return MAPCACHE_FALSE; + return MAPCACHE_LOCK_LOCKED; } else { /* we acquired the lock */ + char *pid_s; + pid_t pid; + apr_size_t pid_s_len; + pid = getpid(); + pid_s = apr_psprintf(ctx->pool,"%"APR_PID_T_FMT,pid); + pid_s_len = strlen(pid_s); + apr_file_write(lockfile,pid_s,&pid_s_len); apr_file_close(lockfile); - return MAPCACHE_TRUE; + return MAPCACHE_LOCK_AQUIRED; } } -void mapcache_unlock_resource(mapcache_context *ctx, char *resource) +mapcache_lock_result mapcache_locker_disk_ping_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { + apr_finfo_t info; + apr_status_t rv; + char *lockname; + lockname = (char*)lock; + rv = apr_stat(&info,lockname,0,ctx->pool); + if(APR_STATUS_IS_ENOENT(rv)) { + return MAPCACHE_LOCK_NOENT; + } else { + return MAPCACHE_LOCK_LOCKED; + } +} + +void mapcache_locker_disk_release_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { - char *lockname = lock_filename_for_resource(ctx,resource); + char *lockname = (char*)lock; apr_file_remove(lockname,ctx->pool); } +void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, void *lock) { + locker->release_lock(ctx, locker, lock); +} + +void mapcache_locker_disk_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { + mapcache_locker_disk *ldisk = (mapcache_locker_disk*)self; + ezxml_t node; + if((node = ezxml_child(doc,"directory")) != NULL) { + ldisk->dir = apr_pstrdup(ctx->pool, node->txt); + } else { + ldisk->dir = apr_pstrdup(ctx->pool,"/tmp"); + } +} + +mapcache_locker* mapcache_locker_disk_create(mapcache_context *ctx) { + mapcache_locker_disk *ld = (mapcache_locker_disk*)apr_pcalloc(ctx->pool, sizeof(mapcache_locker_disk)); + mapcache_locker *l = (mapcache_locker*)ld; + l->type = MAPCACHE_LOCKER_DISK; + l->aquire_lock = mapcache_locker_disk_aquire_lock; + l->parse_xml = mapcache_locker_disk_parse_xml; + l->release_lock = mapcache_locker_disk_release_lock; + l->ping_lock = mapcache_locker_disk_ping_lock; + return l; +} + +struct mapcache_locker_fallback_lock { + mapcache_locker *locker; /*the locker that actually acquired the lock*/ + void *lock; /*the opaque lock returned by the locker*/ +}; + +void mapcache_locker_fallback_release_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { + struct mapcache_locker_fallback_lock *flock = lock; + flock->locker->release_lock(ctx,flock->locker,flock->lock); +} + +mapcache_lock_result mapcache_locker_fallback_ping_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { + struct mapcache_locker_fallback_lock *flock = lock; + return flock->locker->ping_lock(ctx,flock->locker,flock->lock); +} + +mapcache_lock_result mapcache_locker_fallback_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { + int i; + mapcache_locker_fallback *locker = (mapcache_locker_fallback*)self; + struct mapcache_locker_fallback_lock *fallback_lock = apr_pcalloc(ctx->pool, sizeof(struct mapcache_locker_fallback_lock)); + *lock = fallback_lock; + for(i=0;ilockers->nelts;i++) { + mapcache_lock_result lock_result; + mapcache_locker *child_locker = APR_ARRAY_IDX(locker->lockers, i, mapcache_locker*); + void *error; + ctx->pop_errors(ctx,&error); + lock_result = child_locker->aquire_lock(ctx, child_locker, resource, &(fallback_lock->lock)); + if(!GC_HAS_ERROR(ctx)) { + fallback_lock->locker = child_locker; + ctx->push_errors(ctx,error); + return lock_result; + } else { + /*clear the current error if we still have a fallback lock to try */ + if(ilockers->nelts-1) { + ctx->clear_errors(ctx); + } + } + ctx->push_errors(ctx,error); + } + return MAPCACHE_LOCK_NOENT; +} + + +void mapcache_locker_fallback_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { + mapcache_locker_fallback *lm = (mapcache_locker_fallback*)self; + ezxml_t node; + lm->lockers = apr_array_make(ctx->pool,2,sizeof(mapcache_locker*)); + for(node = ezxml_child(doc,"locker"); node; node = node->next) { + mapcache_locker *child_locker; + mapcache_config_parse_locker(ctx,node,&child_locker); + GC_CHECK_ERROR(ctx); + APR_ARRAY_PUSH(lm->lockers,mapcache_locker*) = child_locker; + } +} + +#ifdef USE_MEMCACHE + +#include + +typedef struct { + char *host; + int port; +} mapcache_locker_memcache_server; + +typedef struct { + mapcache_locker locker; + int nservers; + mapcache_locker_memcache_server *servers; +} mapcache_locker_memcache; + +void mapcache_locker_memcache_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + ezxml_t node,server_node; + char *endptr; + for(server_node = ezxml_child(doc,"server"); server_node; server_node = server_node->next) { + lm->nservers++; + } + lm->servers = apr_pcalloc(ctx->pool, lm->nservers * sizeof(mapcache_locker_memcache_server)); + lm->nservers = 0; + for(server_node = ezxml_child(doc,"server"); server_node; server_node = server_node->next) { + if((node = ezxml_child(server_node,"host")) != NULL) { + lm->servers[lm->nservers].host = apr_pstrdup(ctx->pool, node->txt); + } else { + ctx->set_error(ctx, 400, "memcache locker: no provided"); + return; + } + + if((node = ezxml_child(server_node,"port")) != NULL) { + lm->servers[lm->nservers].port = (unsigned int)strtol(node->txt,&endptr,10); + if(*endptr != 0 || lm->servers[lm->nservers].port <= 0) { + ctx->set_error(ctx, 400, "failed to parse memcache locker port \"%s\". Expecting a positive integer", + node->txt); + return; + } + } else { + /* default memcached port */ + lm->servers[lm->nservers].port = 11211; + } + lm->nservers++; + } +} + +static char* memcache_key_for_resource(mapcache_context *ctx, mapcache_locker_memcache *lm, const char *resource) +{ + char *saferes = apr_pstrdup(ctx->pool,resource); + char *safeptr = saferes; + while(*safeptr) { + if(*safeptr==' ' || *safeptr == '/' || *safeptr == '~' || *safeptr == '.' || + *safeptr == '\r' || *safeptr == '\n' || *safeptr == '\t' || *safeptr == '\f' || *safeptr == '\e' || *safeptr == '\a' || *safeptr == '\b') { + *safeptr = '#'; + } + safeptr++; + } + return apr_psprintf(ctx->pool,MAPCACHE_LOCKFILE_PREFIX"%s.lck",saferes); +} + +apr_memcache_t* create_memcache(mapcache_context *ctx, mapcache_locker_memcache *lm) { + apr_status_t rv; + apr_memcache_t *memcache; + char errmsg[120]; + int i; + if(APR_SUCCESS != apr_memcache_create(ctx->pool, lm->nservers, 0, &memcache)) { + ctx->set_error(ctx,500,"memcache locker: failed to create memcache backend"); + return NULL; + } + + for(i=0;inservers;i++) { + apr_memcache_server_t *server; + rv = apr_memcache_server_create(ctx->pool,lm->servers[i].host,lm->servers[i].port,1,1,1,10000,&server); + if(APR_SUCCESS != rv) { + ctx->set_error(ctx,500,"memcache locker: failed to create server %s:%d: %s",lm->servers[i].host,lm->servers[i].port, apr_strerror(rv,errmsg,120)); + return NULL; + } + + rv = apr_memcache_add_server(memcache,server); + if(APR_SUCCESS != rv) { + ctx->set_error(ctx,500,"memcache locker: failed to add server %s:%d: %s",lm->servers[i].host,lm->servers[i].port, apr_strerror(rv,errmsg,120)); + return NULL; + } + } + return memcache; +} + +typedef struct { + apr_memcache_t *memcache; + char *lockname; +} mapcache_lock_memcache; + +mapcache_lock_result mapcache_locker_memcache_ping_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { + apr_status_t rv; + char *one; + size_t ione; + mapcache_lock_memcache *mlock = (mapcache_lock_memcache*)lock; + if(!mlock || !mlock->lockname || !mlock->memcache) + return MAPCACHE_LOCK_NOENT; + rv = apr_memcache_getp(mlock->memcache,ctx->pool,mlock->lockname,&one,&ione,NULL); + if(rv == APR_SUCCESS) + return MAPCACHE_LOCK_LOCKED; + else + return MAPCACHE_LOCK_NOENT; +} + + +mapcache_lock_result mapcache_locker_memcache_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { + apr_status_t rv; + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + char errmsg[120]; + mapcache_lock_memcache *mlock = apr_pcalloc(ctx->pool, sizeof(mapcache_lock_memcache)); + mlock->lockname = memcache_key_for_resource(ctx, lm, resource); + mlock->memcache = create_memcache(ctx,lm); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_LOCK_NOENT; + } + *lock = mlock; + rv = apr_memcache_add(mlock->memcache,mlock->lockname,"1",1,self->timeout,0); + if( rv == APR_SUCCESS) { + return MAPCACHE_LOCK_AQUIRED; + } else if ( rv == APR_EEXIST ) { + return MAPCACHE_LOCK_LOCKED; + } else { + ctx->set_error(ctx,500,"failed to lock resource %s to memcache locker: %s",resource, apr_strerror(rv,errmsg,120)); + return MAPCACHE_LOCK_NOENT; + } +} + +void mapcache_locker_memcache_release_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { + apr_status_t rv; + mapcache_lock_memcache *mlock = (mapcache_lock_memcache*)lock; + char errmsg[120]; + if(!mlock || !mlock->memcache || !mlock->lockname) { + /*error*/ + return; + } + + rv = apr_memcache_delete(mlock->memcache,mlock->lockname,0); + if(rv != APR_SUCCESS && rv!= APR_NOTFOUND) { + ctx->set_error(ctx,500,"memcache: failed to delete key %s: %s", mlock->lockname, apr_strerror(rv,errmsg,120)); + } + +} + +mapcache_locker* mapcache_locker_memcache_create(mapcache_context *ctx) { + mapcache_locker_memcache *lm = (mapcache_locker_memcache*)apr_pcalloc(ctx->pool, sizeof(mapcache_locker_memcache)); + mapcache_locker *l = (mapcache_locker*)lm; + l->type = MAPCACHE_LOCKER_MEMCACHE; + l->aquire_lock = mapcache_locker_memcache_aquire_lock; + l->ping_lock = mapcache_locker_memcache_ping_lock; + l->parse_xml = mapcache_locker_memcache_parse_xml; + l->release_lock = mapcache_locker_memcache_release_lock; + lm->nservers = 0; + lm->servers = NULL; + return l; +} + +#endif + +mapcache_locker* mapcache_locker_fallback_create(mapcache_context *ctx) { + mapcache_locker_fallback *lm = (mapcache_locker_fallback*)apr_pcalloc(ctx->pool, sizeof(mapcache_locker_fallback)); + mapcache_locker *l = (mapcache_locker*)lm; + l->type = MAPCACHE_LOCKER_FALLBACK; + l->aquire_lock = mapcache_locker_fallback_aquire_lock; + l->ping_lock = mapcache_locker_fallback_ping_lock; + l->parse_xml = mapcache_locker_fallback_parse_xml; + l->release_lock = mapcache_locker_fallback_release_lock; + return l; +} + +void mapcache_config_parse_locker_old(mapcache_context *ctx, ezxml_t doc, mapcache_cfg *config) { + /* backwards compatibility */ + int micro_retry; + ezxml_t node; + mapcache_locker_disk *ldisk; + config->locker = mapcache_locker_disk_create(ctx); + ldisk = (mapcache_locker_disk*)config->locker; + if((node = ezxml_child(doc,"lock_dir")) != NULL) { + ldisk->dir = apr_pstrdup(ctx->pool, node->txt); + } else { + ldisk->dir = apr_pstrdup(ctx->pool,"/tmp"); + } + + if((node = ezxml_child(doc,"lock_retry")) != NULL) { + char *endptr; + micro_retry = strtol(node->txt,&endptr,10); + if(*endptr != 0 || micro_retry <= 0) { + ctx->set_error(ctx, 400, "failed to parse lock_retry microseconds \"%s\". Expecting a positive integer", + node->txt); + return; + } + } else { + /* default retry interval is 1/100th of a second, i.e. 10000 microseconds */ + micro_retry = 10000; + } + config->locker->retry_interval = micro_retry / 1000000.0; + config->locker->timeout=120; +} + + +void mapcache_config_parse_locker(mapcache_context *ctx, ezxml_t node, mapcache_locker **locker) { + ezxml_t cur_node; + const char *ltype = ezxml_attr(node, "type"); + if(!ltype) ltype = "disk"; + if(!strcmp(ltype,"disk")) { + *locker = mapcache_locker_disk_create(ctx); + } else if(!strcmp(ltype,"fallback")) { + *locker = mapcache_locker_fallback_create(ctx); + } else if(!strcmp(ltype,"memcache")) { +#ifdef USE_MEMCACHE + *locker = mapcache_locker_memcache_create(ctx); +#else + ctx->set_error(ctx,400,": type \"memcache\" cannot be used as memcache support is not compiled in"); + return; +#endif + } else { + ctx->set_error(ctx,400,": unknown type \"%s\" (allowed are disk and memcache)",ltype); + return; + } + (*locker)->parse_xml(ctx, *locker, node); + + if((cur_node = ezxml_child(node,"retry")) != NULL) { + char *endptr; + (*locker)->retry_interval = strtod(cur_node->txt,&endptr); + if(*endptr != 0 || (*locker)->retry_interval <= 0) { + ctx->set_error(ctx, 400, "failed to locker parse retry seconds \"%s\". Expecting a positive floating point number", + cur_node->txt); + return; + } + } else { + /* default retry interval is 1/10th of a second */ + (*locker)->retry_interval = 0.1; + } + if((cur_node = ezxml_child(node,"timeout")) != NULL) { + char *endptr; + (*locker)->timeout = strtod(cur_node->txt,&endptr); + if(*endptr != 0 || (*locker)->timeout <= 0) { + ctx->set_error(ctx, 400, "failed to parse locker timeout seconds \"%s\". Expecting a positive floating point number", + cur_node->txt); + return; + } + } else { + /* default timeout is 2 minutes */ + (*locker)->timeout = 120; + } +} /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/lib/service_demo.c mapcache-1.6.1/lib/service_demo.c --- mapcache-1.2.1/lib/service_demo.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_demo.c 2017-09-29 21:31:13.000000000 +0000 @@ -29,12 +29,11 @@ #include #include "mapcache.h" +#include "mapcache_services.h" #include #include #include -/** \addtogroup services */ -/** @{ */ static char *demo_head = "\n" @@ -282,7 +281,7 @@ " return null;\n" " }\n" "\n" - " // repeat accross x-axis\n" + " // repeat across x-axis\n" " if (x < 0 || x >= tileRange) {\n" " x = (x % tileRange + tileRange) % tileRange;\n" " }\n" @@ -507,7 +506,7 @@ resolutions = apr_psprintf(ctx->pool,"%s,%.20f",resolutions,grid->levels[i]->resolution); } - if(!tileset->timedimension) { + if(!tileset->dimensions) { ol_layer_name = apr_psprintf(ctx->pool, "%s_%s", tileset->name, grid->name); /* normalize name to something that is a valid variable name */ for(i=0; ipool,"%s%s",caps,ol_layer); } } else { - int id; - apr_array_header_t *timedimvals = tileset->timedimension->get_all_entries( - ctx,tileset->timedimension,tileset); - for(id=0;idnelts;id++) { - if(id>1) break; - char *idval = APR_ARRAY_IDX(timedimvals,id,char*); - char *dimparam_wms = " %s_wms_layer.mergeNewParams({%s:\"%s\"});\n"; - char *dimparam_singletile = " %s_slayer.mergeNewParams({%s:\"%s\"});\n"; - ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s", tileset->name, grid->name, idval); - /* normalize name to something that is a valid variable name */ - for(i=0; ipool, demo_layer_wms, - ol_layer_name, - tileset->name, - grid->name, - apr_pstrcat(ctx->pool,url_prefix,"?",NULL), - tileset->name, - resolutions, - unit, - grid->extent.minx, - grid->extent.miny, - grid->extent.maxx, - grid->extent.maxy, - grid->srs, - smerc, - ol_layer_name); - caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); - caps = apr_psprintf(ctx->pool,"%s%s",caps, - apr_psprintf(ctx->pool,dimparam_wms,ol_layer_name,tileset->timedimension->key,idval)); - - if(service->getmap_strategy == MAPCACHE_GETMAP_ASSEMBLE) { - ol_layer = apr_psprintf(ctx->pool, demo_layer_singletile, + int dim; + for(dim=0;dimdimensions->nelts;dim++) { + int id; + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,dim,mapcache_dimension*); + apr_array_header_t *dimvals = dimension->get_all_entries(ctx,dimension,tileset,NULL,NULL); + for(id=0;idnelts;id++) { + char *idval; + char *dimparam_wms; + char *dimparam_singletile; + if(id>1) break; + idval = APR_ARRAY_IDX(dimvals,id,char*); + dimparam_wms = " %s_wms_layer.mergeNewParams({%s:\"%s\"});\n"; + dimparam_singletile = " %s_slayer.mergeNewParams({%s:\"%s\"});\n"; + ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s_%s", tileset->name, grid->name, dimension->name, idval); + /* normalize name to something that is a valid variable name */ + for(i=0; ipool, demo_layer_wms, ol_layer_name, tileset->name, grid->name, @@ -600,7 +584,28 @@ ol_layer_name); caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); caps = apr_psprintf(ctx->pool,"%s%s",caps, - apr_psprintf(ctx->pool,dimparam_singletile,ol_layer_name,tileset->timedimension->key,idval)); + apr_psprintf(ctx->pool,dimparam_wms,ol_layer_name,dimension->name,idval)); + + if(service->getmap_strategy == MAPCACHE_GETMAP_ASSEMBLE) { + ol_layer = apr_psprintf(ctx->pool, demo_layer_singletile, + ol_layer_name, + tileset->name, + grid->name, + apr_pstrcat(ctx->pool,url_prefix,"?",NULL), + tileset->name, + resolutions, + unit, + grid->extent.minx, + grid->extent.miny, + grid->extent.maxx, + grid->extent.maxy, + grid->srs, + smerc, + ol_layer_name); + caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); + caps = apr_psprintf(ctx->pool,"%s%s",caps, + apr_psprintf(ctx->pool,dimparam_singletile,ol_layer_name,dimension->name,idval)); + } } } } @@ -660,15 +665,11 @@ tileindex_index = apr_hash_first(ctx->pool,ctx->config->tilesets); while(tileindex_index) { int i,j; - char *extension; mapcache_tileset *tileset; const void *key; apr_ssize_t keylen; apr_hash_this(tileindex_index,&key,&keylen,(void**)&tileset); - extension = "png"; - if (tileset->format && tileset->format->extension) - extension = tileset->format->extension; for(j=0; jgrid_links->nelts; j++) { char *resolutions=""; char *unit="dd"; @@ -841,7 +842,7 @@ resolutions = apr_psprintf(ctx->pool,"%s,%.20f",resolutions,grid->levels[i]->resolution); } - if(!tileset->timedimension) { + if(!tileset->dimensions) { ol_layer_name = apr_psprintf(ctx->pool, "%s_%s", tileset->name, grid->name); /* normalize name to something that is a valid variable name */ for(i=0; ipool,"%s%s",caps,ol_layer); } else { - int id; - apr_array_header_t *timedimvals = tileset->timedimension->get_all_entries( - ctx,tileset->timedimension,tileset); - GC_CHECK_ERROR(ctx); - for(id=0;idnelts;id++) { - if(id>1) break; /* we don't want all the entries in the demo interface */ - char *idval = APR_ARRAY_IDX(timedimvals,id,char*); - char *dimparam = "%s_wmts_layer.mergeNewParams({%s:\"%s\"});\n"; - ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s", tileset->name, grid->name, idval); - /* normalize name to something that is a valid variable name */ - for(i=0; ipool, demo_layer_wmts, - ol_layer_name, - tileset->name, - grid->name, - apr_pstrcat(ctx->pool,url_prefix,"wmts/",NULL), - tileset->name, - grid->name, - mime_type, - resolutions, - grid_link->minz, - unit, - grid->extent.minx, - grid->extent.miny, - grid->extent.maxx, - grid->extent.maxy, - grid->srs, - smerc, - ol_layer_name); - caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); - caps = apr_psprintf(ctx->pool,"%s%s",caps, - apr_psprintf(ctx->pool,dimparam,ol_layer_name,tileset->timedimension->key,idval)); - + int dim; + for(dim=0;dimdimensions->nelts;dim++) { + int id; + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,dim,mapcache_dimension*); + apr_array_header_t *dimvals = dimension->get_all_entries(ctx,dimension,tileset,NULL,NULL); + GC_CHECK_ERROR(ctx); + for(id=0;idnelts;id++) { + char *idval; + char *dimparam; + if(id>1) break; /* we don't want all the entries in the demo interface */ + idval = APR_ARRAY_IDX(dimvals,id,char*); + dimparam = "%s_wmts_layer.mergeNewParams({%s:\"%s\"});\n"; + ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s_%s", tileset->name, grid->name, dimension->name, idval); + /* normalize name to something that is a valid variable name */ + for(i=0; ipool, demo_layer_wmts, + ol_layer_name, + tileset->name, + grid->name, + apr_pstrcat(ctx->pool,url_prefix,"wmts/",NULL), + tileset->name, + grid->name, + mime_type, + resolutions, + grid_link->minz, + unit, + grid->extent.minx, + grid->extent.miny, + grid->extent.maxx, + grid->extent.maxy, + grid->srs, + smerc, + ol_layer_name); + caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); + caps = apr_psprintf(ctx->pool,"%s%s",caps, + apr_psprintf(ctx->pool,dimparam,ol_layer_name,dimension->name,idval)); + + } } } } diff -Nru mapcache-1.2.1/lib/service_kml.c mapcache-1.6.1/lib/service_kml.c --- mapcache-1.2.1/lib/service_kml.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_kml.c 2017-09-29 21:31:13.000000000 +0000 @@ -29,8 +29,10 @@ #include "mapcache.h" #include +#include "mapcache_services.h" #include + /** \addtogroup services */ /** @{ */ @@ -72,7 +74,7 @@ t->x = i; t->y = j; t->z = 0; - mapcache_grid_get_extent(ctx, t->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, t->grid_link->grid, t->x, t->y, t->z, &bb); caps = apr_psprintf(ctx->pool, "%s" @@ -102,7 +104,7 @@ } else { mapcache_extent bbox; - mapcache_grid_get_extent(ctx, request->tile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, request->tile->grid_link->grid, request->tile->x, request->tile->y, request->tile->z, &bbox); @@ -144,7 +146,7 @@ t->x = (request->tile->x << 1) + i; t->y = (request->tile->y << 1) + j; t->z = request->tile->z + 1; - mapcache_grid_get_extent(ctx, t->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, t->grid_link->grid, t->x, t->y, t->z, &bb); caps = apr_psprintf(ctx->pool, "%s" diff -Nru mapcache-1.2.1/lib/service_mapguide.c mapcache-1.6.1/lib/service_mapguide.c --- mapcache-1.2.1/lib/service_mapguide.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_mapguide.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,12 +30,12 @@ #include "mapcache.h" #include #include +#include "mapcache_services.h" /** \addtogroup services */ /** @{ */ - void _create_capabilities_mg(mapcache_context *ctx, mapcache_request_get_capabilities *req, char *url, char *path_info, mapcache_cfg *cfg) { ctx->set_error(ctx, 501, "mapguide service does not support capapbilities"); @@ -124,7 +124,7 @@ if(index == 5) { char *gridname; mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc(ctx->pool,sizeof(mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; gridname = sTileset; /*hijack the char* pointer while counting the number of commas */ while(*gridname) { if(*gridname == ';') req->ntiles++; diff -Nru mapcache-1.2.1/lib/services.c mapcache-1.6.1/lib/services.c --- mapcache-1.2.1/lib/services.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/services.c 2017-09-29 21:31:13.000000000 +0000 @@ -70,6 +70,15 @@ ctx->set_error(ctx,404,"unknown service %s",pathinfo); } +int mapcache_config_services_enabled(mapcache_context *ctx, mapcache_cfg *config) { + int count=0,i; + for(i=0; iservices[i]) + count++; + } + return count; +} + /** @} */ diff -Nru mapcache-1.2.1/lib/service_tms.c mapcache-1.6.1/lib/service_tms.c --- mapcache-1.2.1/lib/service_tms.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_tms.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,7 +30,8 @@ #include "mapcache.h" #include #include -#include +#include "mapcache_services.h" + /** \addtogroup services */ /** @{ */ @@ -174,6 +175,133 @@ } +struct requested_tms_layer{ + mapcache_tileset *tileset; + mapcache_grid_link *grid_link; + apr_array_header_t *dimensions; +}; + +/** + * @brief parse a TMS key representing a layer + * @param ctx + * @param key the string representing the requested layer. can be of the form + * layer + * layer@gridname + * layer[dim1=value1] + * layer[dim1=val1][dim2=val2] + * layer[dim=val]@gridname + * @return + */ +static struct requested_tms_layer* _mapcache_service_tms_parse_layer_key(mapcache_context *ctx, char *key) { + char *grid_name = NULL; + struct requested_tms_layer *rtl = apr_pcalloc(ctx->pool, sizeof(struct requested_tms_layer)); + char *at_ptr = strchr(key,'@'); + char *dim_ptr = strchr(key,'['); + if(!at_ptr && !dim_ptr) { + rtl->tileset = mapcache_configuration_get_tileset(ctx->config,key); + if(!rtl->tileset) { + ctx->set_error(ctx,400,"received TMS with invalid layer name"); + return NULL; + } + return rtl; + } else { + key = apr_pstrdup(ctx->pool,key); + if(at_ptr) { + at_ptr = strchr(key,'@'); + *at_ptr = 0; + } + if(dim_ptr) { + dim_ptr = strchr(key,'['); + *dim_ptr = 0; + } + rtl->tileset = mapcache_configuration_get_tileset(ctx->config,key); + if(!rtl->tileset) { + ctx->set_error(ctx,400,"received TMS with invalid layer name"); + return NULL; + } + /*reapply opening bracket*/ + if(dim_ptr) { + *dim_ptr = '['; + } + } + + /* if here, we have a valid rtl->tileset defined */ + + /* if we have a specific grid requested */ + if(at_ptr) { + int i; + grid_name = at_ptr + 1; + if(!*grid_name) { + ctx->set_error(ctx,400,"received invalid tms layer name. expecting layer_name@grid_name"); + return NULL; + } + for(i=0; itileset->grid_links->nelts; i++) { + mapcache_grid_link *sgrid = APR_ARRAY_IDX(rtl->tileset->grid_links,i,mapcache_grid_link*); + if(!strcmp(sgrid->grid->name,grid_name)) { + rtl->grid_link = sgrid; + break; + } + } + if(!rtl->grid_link) { + ctx->set_error(ctx,400,"received invalid tms layer. grid not configured for requested layer"); + return NULL; + } + } + + /*if we have dimensions requested*/ + if(dim_ptr) { + int i; + if(!rtl->tileset->dimensions || rtl->tileset->dimensions->nelts < 1) { + ctx->set_error(ctx,400,"received invalid tms layer. no dimensions configured for tileset"); + return NULL; + } + for(i=0;itileset->dimensions->nelts;i++) { + mapcache_dimension *dimension = APR_ARRAY_IDX(rtl->tileset->dimensions,i,mapcache_dimension*); + char *individual_dim_ptr; + char *dim_lookup = apr_pstrcat(ctx->pool,"[",dimension->name,"=",NULL); + individual_dim_ptr = strstr(dim_ptr,dim_lookup); + if(individual_dim_ptr) { + mapcache_requested_dimension rdim; + char *dim_value; + char *dim_value_end; + if(!rtl->dimensions) { + rtl->dimensions = apr_array_make(ctx->pool,1,sizeof(mapcache_requested_dimension)); + } + dim_value = individual_dim_ptr + strlen(dim_lookup); + if(!*dim_value || *dim_value == ']') { + ctx->set_error(ctx,400,"received invalid tms layer. failed (1) to parse dimension value"); + return NULL; + } + dim_value_end = strchr(dim_value,']'); + if(!dim_value_end) { + ctx->set_error(ctx,400,"received invalid tms layer. failed (2) to parse dimension value"); + return NULL; + } + *dim_value_end = 0; + rdim.dimension = dimension; + rdim.requested_value = apr_pstrdup(ctx->pool,dim_value); + *dim_value_end = ']'; + APR_ARRAY_PUSH(rtl->dimensions,mapcache_requested_dimension) = rdim; + } + } + /* sanity check: make sure we have as many requested dimensions as '[' characters in key */ + { + int i, count; + if(!rtl->dimensions) { + ctx->set_error(ctx,400,"received invalid tms layer. failed (3) to parse dimension values"); + return NULL; + } + for (i=0, count=0; dim_ptr[i]; i++) + count += (dim_ptr[i] == '['); + if(count != rtl->dimensions->nelts) { + ctx->set_error(ctx,400,"received invalid tms layer. failed (4) to parse dimension values"); + return NULL; + } + } + } + return rtl; +} + /** * \brief parse a TMS request * \private \memberof mapcache_service_tms @@ -185,8 +313,6 @@ int index = 0; char *last, *key, *endptr; char *sTileset = NULL; - mapcache_tileset *tileset = NULL; - mapcache_grid_link *grid_link = NULL; char *pathinfo = NULL; int x=-1,y=-1,z=-1; @@ -238,93 +364,73 @@ } } if(index == 5) { - char *gridname; + char *iter,*gridname=NULL; mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc(ctx->pool,sizeof(mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; - gridname = sTileset; /*hijack the char* pointer while counting the number of commas */ - while(*gridname) { - if(*gridname == ';') req->ntiles++; - gridname++; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; + iter = sTileset; + while(*iter) { + if(*iter == ';') req->ntiles++; + iter++; } req->tiles = (mapcache_tile**)apr_pcalloc(ctx->pool,(req->ntiles+1) * sizeof(mapcache_tile*)); - /* reset the hijacked variables back to default value */ - gridname = NULL; req->ntiles = 0; for (key = apr_strtok(sTileset, ";", &last); key != NULL; key = apr_strtok(NULL, ";", &last)) { - tileset = mapcache_configuration_get_tileset(config,key); - if(!tileset) { - /*tileset not found directly, test if it was given as "name@grid" notation*/ - char *tname = apr_pstrdup(ctx->pool,key); - char *gname = tname; - int i; - while(*gname) { - if(*gname == '@') { - *gname = '\0'; - gname++; - break; - } - gname++; - } - if(!gname) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", key); - return; - } - tileset = mapcache_configuration_get_tileset(config,tname); - if(!tileset) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", tname); - return; - } - for(i=0; igrid_links->nelts; i++) { - mapcache_grid_link *sgrid = APR_ARRAY_IDX(tileset->grid_links,i,mapcache_grid_link*); - if(!strcmp(sgrid->grid->name,gname)) { - grid_link = sgrid; - break; - } - } - if(!grid_link) { - ctx->set_error(ctx,404, "received tms request with invalid grid %s", gname); - return; - } - - } else { - grid_link = APR_ARRAY_IDX(tileset->grid_links,0,mapcache_grid_link*); + struct requested_tms_layer *rtl = _mapcache_service_tms_parse_layer_key(ctx,key); + GC_CHECK_ERROR(ctx); + if(!rtl->grid_link) { + rtl->grid_link = APR_ARRAY_IDX(rtl->tileset->grid_links,0,mapcache_grid_link*); } + + /*make sure all our requested layers have the same grid*/ if(!gridname) { - gridname = grid_link->grid->name; + gridname = rtl->grid_link->grid->name; } else { - if(strcmp(gridname,grid_link->grid->name)) { + if(strcmp(gridname,rtl->grid_link->grid->name)) { ctx->set_error(ctx,400,"received tms request with conflicting grids %s and %s", - gridname,grid_link->grid->name); + gridname,rtl->grid_link->grid->name); return; } } if(((mapcache_service_tms*)this)->reverse_y) { - y = grid_link->grid->levels[z]->maxy - y - 1; + y = rtl->grid_link->grid->levels[z]->maxy - y - 1; } - req->tiles[req->ntiles] = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); - switch(grid_link->grid->origin) { + req->tiles[req->ntiles] = mapcache_tileset_tile_create(ctx->pool, rtl->tileset, rtl->grid_link); + switch(rtl->grid_link->grid->origin) { case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: req->tiles[req->ntiles]->x = x; req->tiles[req->ntiles]->y = y; break; case MAPCACHE_GRID_ORIGIN_TOP_LEFT: req->tiles[req->ntiles]->x = x; - req->tiles[req->ntiles]->y = grid_link->grid->levels[z]->maxy - y - 1; + req->tiles[req->ntiles]->y = rtl->grid_link->grid->levels[z]->maxy - y - 1; break; case MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT: - req->tiles[req->ntiles]->x = grid_link->grid->levels[z]->maxx - x - 1; + req->tiles[req->ntiles]->x = rtl->grid_link->grid->levels[z]->maxx - x - 1; req->tiles[req->ntiles]->y = y; break; case MAPCACHE_GRID_ORIGIN_TOP_RIGHT: - req->tiles[req->ntiles]->x = grid_link->grid->levels[z]->maxx - x - 1; - req->tiles[req->ntiles]->y = grid_link->grid->levels[z]->maxy - y - 1; + req->tiles[req->ntiles]->x = rtl->grid_link->grid->levels[z]->maxx - x - 1; + req->tiles[req->ntiles]->y = rtl->grid_link->grid->levels[z]->maxy - y - 1; break; } req->tiles[req->ntiles]->z = z; mapcache_tileset_tile_validate(ctx,req->tiles[req->ntiles]); + if(rtl->dimensions) { + int i; + for(i=0;idimensions->nelts;i++) { + mapcache_requested_dimension rdim = APR_ARRAY_IDX(rtl->dimensions,i,mapcache_requested_dimension); + int j; + for(j=0;jtiles[req->ntiles]->dimensions->nelts;j++) { + mapcache_requested_dimension *tile_dim = APR_ARRAY_IDX(req->tiles[req->ntiles]->dimensions,j,mapcache_requested_dimension*); + if(!strcasecmp(tile_dim->dimension->name,rdim.dimension->name)) { + tile_dim->requested_value = rdim.requested_value; + } + } + } + } req->ntiles++; GC_CHECK_ERROR(ctx); } @@ -335,45 +441,19 @@ ctx->pool,sizeof(mapcache_request_get_capabilities_tms)); req->request.request.type = MAPCACHE_REQUEST_GET_CAPABILITIES; if(index == 2) { - tileset = mapcache_configuration_get_tileset(config,sTileset); - if(!tileset) { - /*tileset not found directly, test if it was given as "name@grid" notation*/ - char *tname = apr_pstrdup(ctx->pool,sTileset); - char *gname = tname; - int i; - while(*gname) { - if(*gname == '@') { - *gname = '\0'; - gname++; - break; - } - gname++; - } - if(!gname) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", key); - return; - } - tileset = mapcache_configuration_get_tileset(config,tname); - if(!tileset) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", tname); - return; - } - for(i=0; igrid_links->nelts; i++) { - mapcache_grid_link *sgrid = APR_ARRAY_IDX(tileset->grid_links,i,mapcache_grid_link*); - if(!strcmp(sgrid->grid->name,gname)) { - grid_link = sgrid; - break; - } - } - if(!grid_link) { - ctx->set_error(ctx,404, "received tms request with invalid grid %s", gname); - return; - } - } else { - grid_link = APR_ARRAY_IDX(tileset->grid_links,0,mapcache_grid_link*); + struct requested_tms_layer *rtl; + if(strchr(sTileset,';')) { + ctx->set_error(ctx,400,"tms caps: invalid tileset name"); + return; + } + + rtl = _mapcache_service_tms_parse_layer_key(ctx,sTileset); + GC_CHECK_ERROR(ctx); + if(!rtl->grid_link) { + rtl->grid_link = APR_ARRAY_IDX(rtl->tileset->grid_links,0,mapcache_grid_link*); } - req->tileset = tileset; - req->grid_link = grid_link; + req->tileset = rtl->tileset; + req->grid_link = rtl->grid_link; } if(index >= 1) { req->version = apr_pstrdup(ctx->pool,"1.0.0"); diff -Nru mapcache-1.2.1/lib/service_ve.c mapcache-1.6.1/lib/service_ve.c --- mapcache-1.2.1/lib/service_ve.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_ve.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,13 +30,13 @@ #include "mapcache.h" #include #include +#include "mapcache_services.h" /** \addtogroup services */ /** @{ */ - void _create_capabilities_ve(mapcache_context *ctx, mapcache_request_get_capabilities *req, char *url, char *path_info, mapcache_cfg *cfg) { ctx->set_error(ctx, 501, "ve service does not support capapbilities"); @@ -98,32 +98,12 @@ quadkey = apr_table_get(params, "tile"); tile = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); if (quadkey) { - z = strlen(quadkey); + mapcache_util_quadkey_decode(ctx, quadkey, &x, &y, &z); + GC_CHECK_ERROR(ctx); if (z < 1 || z >= grid_link->grid->nlevels) { ctx->set_error(ctx, 404, "received ve request with invalid z level %d\n", z); return; } - x = y = 0; - for (i = z; i; i--) { - int mask = 1 << (i - 1); - switch (quadkey[z - i]) { - case '0': - break; - case '1': - x |= mask; - break; - case '2': - y |= mask; - break; - case '3': - x |= mask; - y |= mask; - break; - default: - ctx->set_error(ctx, 404, "Invalid Quadkey sequence"); - return; - } - } } else { ctx->set_error(ctx, 400, "received ve request with no tile quadkey"); return; @@ -131,7 +111,7 @@ req = (mapcache_request_get_tile*) apr_pcalloc(ctx->pool, sizeof (mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; req->ntiles = 1; req->tiles = (mapcache_tile**) apr_pcalloc(ctx->pool, sizeof (mapcache_tile*)); req->tiles[0] = tile; diff -Nru mapcache-1.2.1/lib/service_wms.c mapcache-1.6.1/lib/service_wms.c --- mapcache-1.2.1/lib/service_wms.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_wms.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,7 +30,14 @@ #include "mapcache.h" #include #include -#include "ezxml.h" +#include "mapcache_services.h" + +static int sort_strings(const void* pa, const void* pb) +{ + char** ppszA = (char**)pa; + char** ppszB = (char**)pb; + return strcmp(*ppszA, *ppszB); +} /** \addtogroup services */ /** @{ */ @@ -158,7 +165,7 @@ "\n" */ - tmpxml = ezxml_add_child(capxml,"Exceptions",0); + tmpxml = ezxml_add_child(capxml,"Exception",0); ezxml_set_txt(ezxml_add_child(tmpxml,"Format",0),"text/plain"); vendorxml = ezxml_add_child(capxml,"VendorSpecificCapabilities",0); @@ -173,16 +180,28 @@ * * TODO: check for duplicates in gris srs */ - grid_index = apr_hash_first(ctx->pool,cfg->grids); - while(grid_index) { - const void *key; - apr_ssize_t keylen; - mapcache_grid *grid = NULL; - apr_hash_this(grid_index,&key,&keylen,(void**)&grid); - ezxml_set_txt(ezxml_add_child(toplayer,"SRS",0),grid->srs); - grid_index = apr_hash_next(grid_index); - } + { + int srs_count = (int)apr_hash_count(cfg->grids); + char** srs_list = (char**)malloc(srs_count * sizeof(char*)); + int srs_iter = 0; + int i; + grid_index = apr_hash_first(ctx->pool,cfg->grids); + while(grid_index) { + const void *key; + apr_ssize_t keylen; + mapcache_grid *grid = NULL; + apr_hash_this(grid_index,&key,&keylen,(void**)&grid); + srs_list[srs_iter++] = grid->srs; + grid_index = apr_hash_next(grid_index); + } + qsort(srs_list, srs_count, sizeof(char*), sort_strings); + for(i = 0; i < srs_count; i ++) + { + ezxml_set_txt(ezxml_add_child(toplayer,"SRS",0),srs_list[i]); + } + free(srs_list); + } tileindex_index = apr_hash_first(ctx->pool,cfg->tilesets); @@ -200,7 +219,7 @@ layerxml = ezxml_add_child(toplayer,"Layer",0); ezxml_set_attr(layerxml, "cascaded", "1"); ezxml_set_attr(layerxml, "queryable", (tileset->source && tileset->source->info_formats)?"1":"0"); - + ezxml_set_txt(ezxml_add_child(layerxml,"Name",0),tileset->name); tsxml = ezxml_add_child(vendorxml, "TileSet",0); @@ -228,8 +247,9 @@ if(tileset->dimensions) { for(i=0; idimensions->nelts; i++) { - const char **value; - char *dimval; + apr_array_header_t *values; + int value_idx; + char *dimval = NULL; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); ezxml_t dimxml = ezxml_add_child(layerxml,"Dimension",0); ezxml_set_attr(dimxml,"name",dimension->name); @@ -238,12 +258,14 @@ if(dimension->unit) { ezxml_set_attr(dimxml,"units",dimension->unit); } - value = dimension->print_ogc_formatted_values(ctx,dimension); - dimval = apr_pstrdup(ctx->pool,*value); - value++; - while(*value) { - dimval = apr_pstrcat(ctx->pool,dimval,",",*value,NULL); - value++; + values = dimension->get_all_ogc_formatted_entries(ctx,dimension,tileset,NULL,NULL); + for(value_idx=0;value_idxnelts;value_idx++) { + char *idval = APR_ARRAY_IDX(values,value_idx,char*); + if(dimval) { + dimval = apr_pstrcat(ctx->pool,dimval,",",idval,NULL); + } else { + dimval = apr_pstrdup(ctx->pool,idval); + } } ezxml_set_txt(dimxml,dimval); } @@ -317,6 +339,26 @@ request->request.mime_type = apr_pstrdup(ctx->pool,"text/xml"); } +static char *_lookup_auto_projection(mapcache_context *ctx, const char *srs) { + if(!strcasecmp(srs,"auto:42001")) { + char *srsdup = apr_pstrdup(ctx->pool,srs); + char *lon = strchr(srsdup,','),*lat; + int nLon,nLat; + if(!lon) return srsdup; + lon = strchr(lon+1,','); + if(!lon) return srsdup; + lon++; + lat = strchr(lon,','); + if(!lat) return srsdup; + *lat = 0; + lat++; + nLon = (int)(floor( (atof(lon) + 180.0) / 6.0 ))*6 + 3 - 180; + nLat = (atof(lat)>=0)?45:-45; + return apr_psprintf(ctx->pool,"auto:42001,9001,%d,%d",nLon,nLat); + } + return (char*)srs; +} + /** * \brief parse a WMS request * \private \memberof mapcache_service_wms @@ -475,7 +517,6 @@ char *last, *layers; const char *key; int count=1; - int nallocated = 0; int i,layeridx; int x,y,z; mapcache_request_get_map *map_req = NULL; @@ -483,6 +524,7 @@ mapcache_grid_link *main_grid_link = NULL; mapcache_tileset *main_tileset = NULL; mapcache_request_type type; + mapcache_image_format *imf; /* count the number of layers that are requested. * if we are in combined-mirror mode, then there is @@ -515,6 +557,8 @@ main_tileset->name = (char*)key; } + srs = _lookup_auto_projection(ctx,srs); + for(i=0; igrid_links->nelts; i++) { mapcache_grid_link *sgrid = APR_ARRAY_IDX(main_tileset->grid_links,i,mapcache_grid_link*); /* look for a grid with a matching srs */ @@ -545,23 +589,42 @@ type = MAPCACHE_REQUEST_GET_MAP; } + imf = wms_service->getmap_format; + if(wms_service->allow_format_override) { + str = apr_table_get(params,"FORMAT"); + if(strcmp(str,imf->name) && strcmp(str,imf->mime_type)) { + apr_hash_index_t *hi; + for (hi = apr_hash_first(ctx->pool, ctx->config->image_formats); hi; hi = apr_hash_next(hi)) { + apr_hash_this(hi, NULL, NULL, (void**)&imf); + if(!strcmp(imf->name, str) || (imf->mime_type && !strcmp(imf->mime_type, str))) { + break; + } + } + if(!hi) { /* did not find any matching format for given mimetype or name */ + errcode = 404; + errmsg = apr_psprintf(ctx->pool,"received wms request with invalid format %s", str); + goto proxies; + } + } + } + if(type == MAPCACHE_REQUEST_GET_TILE) { tile_req = apr_pcalloc(ctx->pool, sizeof(mapcache_request_get_tile)); - tile_req->request.type = MAPCACHE_REQUEST_GET_TILE; tile_req->tiles = apr_pcalloc(ctx->pool, count*sizeof(mapcache_tile*)); - tile_req->format = wms_service->getmap_format; + tile_req->image_request.format = imf; *request = (mapcache_request*)tile_req; + (*request)->type = MAPCACHE_REQUEST_GET_TILE; } else { map_req = apr_pcalloc(ctx->pool, sizeof(mapcache_request_get_map)); - map_req->request.type = MAPCACHE_REQUEST_GET_MAP; map_req->maps = apr_pcalloc(ctx->pool, count*sizeof(mapcache_map*)); map_req->getmap_strategy = wms_service->getmap_strategy; map_req->resample_mode = wms_service->resample_mode; - map_req->getmap_format = wms_service->getmap_format; + map_req->image_request.format = imf; *request = (mapcache_request*)map_req; + (*request)->type = MAPCACHE_REQUEST_GET_MAP; } - nallocated = count; + /* * loop through all the layers to verify that they reference the requested grid, @@ -575,7 +638,7 @@ int i; mapcache_tileset *tileset = main_tileset; mapcache_grid_link *grid_link = main_grid_link; - apr_table_t *dimtable = NULL; + apr_array_header_t *dimtable = NULL; if(layeridx) { /* @@ -636,84 +699,25 @@ const char *value; if(tileset->dimensions) { for(i=0; idimensions->nelts; i++) { + char *dim_name; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - if((value = (char*)apr_table_get(params,dimension->name)) != NULL) { - char *tmpval = apr_pstrdup(ctx->pool,value); - int ok = dimension->validate(ctx,dimension,&tmpval); - GC_CHECK_ERROR(ctx); - if(ok == MAPCACHE_SUCCESS) - apr_table_setn(dimtable,dimension->name,tmpval); - else { - errcode = 400; - errmsg = apr_psprintf(ctx->pool, "dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); - goto proxies; - } - } - } - } - if(tileset->timedimension) { - /* possibly duplicate the created map/tile for each entry of the requested time dimension */ - apr_array_header_t *timedim_selected; - value = apr_table_get(params,tileset->timedimension->key); - if(!value) - value = tileset->timedimension->default_value; - timedim_selected = mapcache_timedimension_get_entries_for_value(ctx, - tileset->timedimension, tileset, grid_link->grid, &extent, value); - GC_CHECK_ERROR(ctx); - if(!timedim_selected || timedim_selected->nelts == 0) { - errcode = 404; - errmsg = apr_psprintf(ctx->pool,"no matching entry for given TIME dimension \"%s\" in tileset \"%s\"", - tileset->timedimension->key, tileset->name); - goto proxies; - } - if(type == MAPCACHE_REQUEST_GET_TILE) { - int i; - /* we need to create more tile/map entries */ - if(timedim_selected->nelts > 1) { - /* apr pools have no realloc */ - nallocated = nallocated + timedim_selected->nelts - 1; - mapcache_tile** tmptiles = - apr_palloc(ctx->pool, nallocated * sizeof(mapcache_tile*)); - for(i=0;intiles;i++) { - tmptiles[i] = tile_req->tiles[i]; - } - tile_req->tiles = tmptiles; - /* end realloc workaround */ - } - for(i=0;inelts;i++) { - if(i) { - tile_req->tiles[tile_req->ntiles] = - mapcache_tileset_tile_clone(ctx->pool,tile_req->tiles[tile_req->ntiles-1]); - tile_req->ntiles++; - } - apr_table_set(tile_req->tiles[tile_req->ntiles-1]->dimensions,tileset->timedimension->key, - APR_ARRAY_IDX(timedim_selected,i,char*)); + if(!strcasecmp(dimension->name,"TIME") || !strcasecmp(dimension->name,"ELEVATION")) { + dim_name = dimension->name; + } else { + dim_name = apr_pstrcat(ctx->pool, "dim_", dimension->name, NULL); } - } else { - int i; - /* we need to create more tile/map entries */ - if(timedim_selected->nelts > 1) { - /* apr pools have no realloc */ - nallocated = nallocated + timedim_selected->nelts - 1; - mapcache_map** tmpmaps = - apr_palloc(ctx->pool, nallocated * sizeof(mapcache_map*)); - for(i=0;inmaps;i++) { - tmpmaps[i] = map_req->maps[i]; + if((value = (char*)apr_table_get(params,dim_name)) == NULL) { + if(strcasecmp(dimension->name,"TIME") && strcasecmp(dimension->name,"ELEVATION")) { + /* also test for the dimension without the DIM_ prefix if the latter was not found in the KVP params */ + dim_name = dimension->name; + value = (char*)apr_table_get(params,dim_name); } - map_req->maps = tmpmaps; - /* end realloc workaround */ } - for(i=0;inelts;i++) { - if(i) { - map_req->maps[map_req->nmaps] = - mapcache_tileset_map_clone(ctx->pool,map_req->maps[map_req->nmaps-1]); - map_req->nmaps++; - } - apr_table_set(map_req->maps[map_req->nmaps-1]->dimensions,tileset->timedimension->key, - APR_ARRAY_IDX(timedim_selected,i,char*)); + + if(value) { + mapcache_set_requested_dimension(ctx,dimtable,dimension->name,value); + GC_CHECK_ERROR(ctx); } - } } } @@ -813,17 +817,8 @@ mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); const char *value; if((value = (char*)apr_table_get(params,dimension->name)) != NULL) { - char *tmpval = apr_pstrdup(ctx->pool,value); - int ok = dimension->validate(ctx,dimension,&tmpval); + mapcache_map_set_cached_dimension(ctx,&fi->map,dimension->name,value); GC_CHECK_ERROR(ctx); - if(ok == MAPCACHE_SUCCESS) - apr_table_setn(fi->map.dimensions,dimension->name,tmpval); - else { - errcode = 400; - errmsg = apr_psprintf(ctx->pool,"dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); - goto proxies; - } } } } @@ -865,7 +860,7 @@ for(j=0; jmatch_params->nelts; j++) { mapcache_dimension *match_param = APR_ARRAY_IDX(rule->match_params,j,mapcache_dimension*); const char *value = apr_table_get(params,match_param->name); - if(!value || match_param->validate(ctx,match_param,(char**)&value) == MAPCACHE_FAILURE) { + if(!value || match_param->get_entries_for_value(ctx,match_param,value,NULL,NULL,NULL)->nelts == 0) { /* the parameter was not supplied, or did not validate: we don't apply this rule */ got_a_match = 0; break; @@ -876,7 +871,7 @@ *request = (mapcache_request*)req_proxy; (*request)->service = this; (*request)->type = MAPCACHE_REQUEST_PROXY; - req_proxy->http = rule->http; + req_proxy->rule = rule; req_proxy->params = params; if(rule->append_pathinfo) { req_proxy->pathinfo = pathinfo; @@ -916,32 +911,42 @@ for( rule_node = ezxml_child(node,"forwarding_rule"); rule_node; rule_node = rule_node->next) { mapcache_forwarding_rule *rule; - ezxml_t http_node; - ezxml_t pathinfo_node; - ezxml_t param_node; + ezxml_t node; char *name = (char*)ezxml_attr(rule_node,"name"); if(!name) name = "(null)"; rule = apr_pcalloc(ctx->pool, sizeof(mapcache_forwarding_rule)); rule->name = apr_pstrdup(ctx->pool,name); rule->match_params = apr_array_make(ctx->pool,1,sizeof(mapcache_dimension*)); + rule->max_post_len = 10485760; /* 10 megabytes by default */ - pathinfo_node = ezxml_child(rule_node,"append_pathinfo"); - if(pathinfo_node && !strcasecmp(pathinfo_node->txt,"true")) { + node = ezxml_child(rule_node,"append_pathinfo"); + if(node && !strcasecmp(node->txt,"true")) { rule->append_pathinfo = 1; } else { rule->append_pathinfo = 0; } - http_node = ezxml_child(rule_node,"http"); - if(!http_node) { + + node = ezxml_child(rule_node,"max_post_length"); + if(node) { + char *endptr; + rule->max_post_len= (size_t)strtol(node->txt,&endptr,10); + if(*endptr != 0 || rule->max_post_len <= 0) { + ctx->set_error(ctx,500,"rule \"%s\" cannot have a negative or null ",name); + return; + } + } + + node = ezxml_child(rule_node,"http"); + if(!node) { ctx->set_error(ctx,500,"rule \"%s\" does not contain an block",name); return; } - rule->http = mapcache_http_configuration_parse_xml(ctx,http_node); + rule->http = mapcache_http_configuration_parse_xml(ctx,node); GC_CHECK_ERROR(ctx); - for(param_node = ezxml_child(rule_node,"param"); param_node; param_node = param_node->next) { - char *name = (char*)ezxml_attr(param_node,"name"); - char *type = (char*)ezxml_attr(param_node,"type"); + for(node = ezxml_child(rule_node,"param"); node; node = node->next) { + char *name = (char*)ezxml_attr(node,"name"); + char *type = (char*)ezxml_attr(node,"type"); mapcache_dimension *dimension = NULL; @@ -952,9 +957,9 @@ if(type && *type) { if(!strcmp(type,"values")) { - dimension = mapcache_dimension_values_create(ctx->pool); + dimension = mapcache_dimension_values_create(ctx,ctx->pool); } else if(!strcmp(type,"regex")) { - dimension = mapcache_dimension_regex_create(ctx->pool); + dimension = mapcache_dimension_regex_create(ctx,ctx->pool); } else { ctx->set_error(ctx,400,"unknown type \"%s\". expecting \"values\" or \"regex\".",type); return; @@ -963,10 +968,11 @@ ctx->set_error(ctx,400, "mandatory attribute \"type\" not found in "); return; } + GC_CHECK_ERROR(ctx); dimension->name = apr_pstrdup(ctx->pool,name); - dimension->configuration_parse_xml(ctx,dimension,param_node); + dimension->configuration_parse_xml(ctx,dimension,node); GC_CHECK_ERROR(ctx); APR_ARRAY_PUSH(rule->match_params,mapcache_dimension*) = dimension; @@ -986,11 +992,16 @@ wms->getmap_format = mapcache_configuration_get_image_format(cfg,"JPEG"); if ((rule_node = ezxml_child(node,"format")) != NULL) { + const char *attr; wms->getmap_format = mapcache_configuration_get_image_format(cfg,rule_node->txt); if(!wms->getmap_format) { ctx->set_error(ctx,400, "unknown %s for wms service", rule_node->txt); return; } + attr = ezxml_attr(rule_node,"allow_client_override"); + if(attr && !strcmp(attr,"true")) { + wms->allow_format_override = 1; + } } if ((rule_node = ezxml_child(node,"resample_mode")) != NULL) { @@ -1022,9 +1033,7 @@ \"http://schemas.opengis.net/wms/1.1.1/exception_1_1_1.dtd\">\n\ \n\ \n\ -\n\ \n\ %s\ "; @@ -1041,7 +1050,9 @@ } } - *err_body = apr_psprintf(ctx->pool,template,msg,exceptions); + *err_body = apr_psprintf(ctx->pool,template, + mapcache_util_str_xml_escape(ctx->pool, msg, MAPCACHE_UTIL_XML_SECTION_TEXT), + exceptions); apr_table_set(headers, "Content-Type", "application/vnd.ogc.se_xml"); } @@ -1064,6 +1075,7 @@ service->getmap_strategy = MAPCACHE_GETMAP_ASSEMBLE; service->resample_mode = MAPCACHE_RESAMPLE_BILINEAR; service->getmap_format = NULL; + service->allow_format_override = 0; return (mapcache_service*)service; } diff -Nru mapcache-1.2.1/lib/service_wmts.c mapcache-1.6.1/lib/service_wmts.c --- mapcache-1.2.1/lib/service_wmts.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/service_wmts.c 2017-09-29 21:31:13.000000000 +0000 @@ -31,13 +31,12 @@ #include #include #include -#include "ezxml.h" +#include "mapcache_services.h" /** \addtogroup services */ /** @{ */ - static ezxml_t _wmts_capabilities(mapcache_context *ctx, mapcache_cfg *cfg) { char *schemaLocation = "http://www.opengis.net/wmts/1.0 http://schemas.opengis.net/wmts/1.0/wmtsGetCapabilities_response.xsd"; @@ -348,6 +347,15 @@ ezxml_set_txt(ezxml_add_child(layer,"ows:Abstract",0),abstract); } + if(tileset->wgs84bbox.minx != tileset->wgs84bbox.maxx) { + ezxml_t bbox = ezxml_add_child(layer,"ows:WGS84BoundingBox",0); + ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0), + apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.minx, tileset->wgs84bbox.miny)); + ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0), + apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.maxx, tileset->wgs84bbox.maxy)); + } + + ezxml_set_txt(ezxml_add_child(layer,"ows:Identifier",0),tileset->name); style = ezxml_add_child(layer,"Style",0); @@ -363,8 +371,8 @@ if(tileset->dimensions) { for(i=0; idimensions->nelts; i++) { - const char **values; - const char **value; + apr_array_header_t *values; + int value_idx; mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); ezxml_t dim = ezxml_add_child(layer,"Dimension",0); ezxml_set_txt(ezxml_add_child(dim,"ows:Identifier",0),dimension->name); @@ -373,11 +381,10 @@ if(dimension->unit) { ezxml_set_txt(ezxml_add_child(dim,"UOM",0),dimension->unit); } - values = dimension->print_ogc_formatted_values(ctx,dimension); - value = values; - while(*value) { - ezxml_set_txt(ezxml_add_child(dim,"Value",0),*value); - value++; + values = dimension->get_all_ogc_formatted_entries(ctx,dimension,tileset,NULL,NULL); + for(value_idx=0;value_idxnelts;value_idx++) { + char *idval = APR_ARRAY_IDX(values,value_idx,char*); + ezxml_set_txt(ezxml_add_child(dim,"Value",0),idval); } dimensionstemplate = apr_pstrcat(ctx->pool,dimensionstemplate,"{",dimension->name,"}/",NULL); } @@ -390,13 +397,6 @@ } } - if(tileset->wgs84bbox.minx != tileset->wgs84bbox.maxx) { - ezxml_t bbox = ezxml_add_child(layer,"ows:WGS84BoundingBox",0); - ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0), - apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.minx, tileset->wgs84bbox.miny)); - ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0), - apr_psprintf(ctx->pool,"%f %f",tileset->wgs84bbox.maxx, tileset->wgs84bbox.maxy)); - } for(i=0; igrid_links->nelts; i++) { mapcache_grid_link *grid_link = APR_ARRAY_IDX(tileset->grid_links,i,mapcache_grid_link*); @@ -416,13 +416,13 @@ ezxml_set_txt(ezxml_add_child(matrixlimits,"TileMatrix",0), apr_psprintf(ctx->pool,"%s:%d",grid_link->grid->name,j)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MinTileRow",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].minx)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].miny)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MaxTileRow",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxx-1)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxy-1)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MinTileCol",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].miny)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].minx)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MaxTileCol",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxy-1)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxx-1)); } } @@ -496,10 +496,18 @@ bbox = ezxml_add_child(tmset,"ows:BoundingBox",0); - ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0),apr_psprintf(ctx->pool,"%f %f", - grid->extent.minx, grid->extent.miny)); - ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0),apr_psprintf(ctx->pool,"%f %f", - grid->extent.maxx, grid->extent.maxy)); + if(mapcache_is_axis_inverted(grid->srs)) { + ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.miny, grid->extent.minx)); + ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.maxy, grid->extent.maxx)); + } else { + ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.minx, grid->extent.miny)); + ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.maxx, grid->extent.maxy)); + } + ezxml_set_attr(bbox,"crs",mapcache_grid_get_crs(ctx,grid)); ezxml_set_txt(ezxml_add_child(tmset,"ows:SupportedCRS",0),mapcache_grid_get_crs(ctx,grid)); @@ -526,6 +534,7 @@ break; case MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT: case MAPCACHE_GRID_ORIGIN_TOP_RIGHT: + default: ctx->set_error(ctx,500,"origin not implemented"); return; } @@ -565,8 +574,6 @@ *infoformat = NULL, *fi_i = NULL, *fi_j = NULL; apr_table_t *dimtable = NULL; mapcache_extent *extent = NULL; - char *timedim = NULL; - apr_array_header_t *timedim_selected; /* the individual time entries that corresponded to the input request */ mapcache_tileset *tileset = NULL; int row,col,level,x,y; int kvp = 0; @@ -629,14 +636,6 @@ } } } - if(tileset->timedimension) { - const char* value; - if((value = apr_table_get(params,tileset->timedimension->key)) != NULL) { - timedim = apr_pstrdup(ctx->pool,value); - } else { - timedim = apr_pstrdup(ctx->pool,tileset->timedimension->default_value); - } - } if(!strcasecmp(str,"getfeatureinfo")) { infoformat = apr_table_get(params,"INFOFORMAT"); fi_i = apr_table_get(params,"I"); @@ -701,10 +700,6 @@ continue; } } - if(tileset->timedimension) { - timedim = key; - continue; - } if(!matrixset) { matrixset = key; continue; @@ -767,40 +762,9 @@ return; } - /*validate dimensions*/ - if(tileset->dimensions) { - int i; - if(!dimtable) { - ctx->set_error(ctx,404, "received request with no dimensions"); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","dim"); - return; - } - - for(i=0; idimensions->nelts; i++) { - char *tmpval; - int ok; - mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - const char *value = apr_table_get(dimtable,dimension->name); - if(value) { - tmpval = apr_pstrdup(ctx->pool,value); - ok = dimension->validate(ctx,dimension,&tmpval); - GC_CHECK_ERROR(ctx); - if(ok != MAPCACHE_SUCCESS) { - ctx->set_error(ctx,404,"dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","%s",dimension->name); - return; - } - - /* re-set the eventually modified value in the dimension table */ - apr_table_set(dimtable,dimension->name,tmpval); - } - } - } - if(!matrixset) { ctx->set_error(ctx, 404, "received wmts request with no TILEMATRIXSET"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilematrixset"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileMatrixSet"); return; } else { int i; @@ -812,49 +776,49 @@ } if(!grid_link) { ctx->set_error(ctx, 404, "received wmts request with invalid TILEMATRIXSET %s",matrixset); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilematrixset"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileMatrixSet"); return; } } if(!matrix) { ctx->set_error(ctx, 404, "received wmts request with no TILEMATRIX"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilematrix"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileMatrix"); return; } else { char *endptr; level = (int)strtol(matrix,&endptr,10); if(*endptr != 0 || level < grid_link->minz || level >= grid_link->maxz) { ctx->set_error(ctx, 404, "received wmts request with invalid TILEMATRIX %s", matrix); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilematrix"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileMatrix"); return; } } if(!tilerow) { ctx->set_error(ctx, 404, "received wmts request with no TILEROW"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilerow"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileRow"); return; } else { char *endptr; row = (int)strtol(tilerow,&endptr,10); if(*endptr != 0 || row < 0) { ctx->set_error(ctx, 404, "received wmts request with invalid TILEROW %s",tilerow); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilerow"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileRow"); return; } } if(!tilecol) { ctx->set_error(ctx, 404, "received wmts request with no TILECOL"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilecol"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileCol"); return; } else { char *endptr; col = (int)strtol(tilecol,&endptr,10); if(endptr == tilecol || col < 0) { ctx->set_error(ctx, 404, "received wmts request with invalid TILECOL %s",tilecol); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilecol"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileCol"); return; } } @@ -877,17 +841,19 @@ x = grid_link->grid->levels[level]->maxx - col - 1; y = row; break; + default: + ctx->set_error(ctx,500,"BUG: invalid grid origin"); + return; } - if(fi_j || timedim) { + if(fi_j) { //we need the extent of the request, compute it here extent = apr_pcalloc(ctx->pool, sizeof(mapcache_extent)); - mapcache_grid_get_extent(ctx,grid_link->grid,x,y,level,extent); + mapcache_grid_get_tile_extent(ctx,grid_link->grid,x,y,level,extent); } if(!fi_j) { /*we have a getTile request*/ - int i; #ifdef PEDANTIC_WMTS_FORMAT_CHECK if(tileset->format) { @@ -913,58 +879,47 @@ mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc( ctx->pool,sizeof(mapcache_request_get_tile)); - req->request.type = MAPCACHE_REQUEST_GET_TILE; - if(timedim) { - timedim_selected = mapcache_timedimension_get_entries_for_value(ctx, - tileset->timedimension, tileset, grid_link->grid, extent, timedim); - GC_CHECK_ERROR(ctx); - if(!timedim_selected || timedim_selected->nelts == 0) { - ctx->set_error(ctx, 404, "no matching entry for given TIME dimension"); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TIME"); - return; - } - req->ntiles = timedim_selected->nelts; - } else { - req->ntiles = 1; - } + ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; + req->ntiles = 1; req->tiles = (mapcache_tile**)apr_pcalloc(ctx->pool,req->ntiles * sizeof(mapcache_tile*)); - for(i=0;intiles;i++) { - req->tiles[i] = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); - if(!req->tiles[i]) { - ctx->set_error(ctx, 500, "failed to allocate tile"); - if(kvp) ctx->set_exception(ctx,"NoApplicableCode",""); - return; - } - - /*populate dimensions*/ - if(tileset->dimensions) { - int d; - for(d=0; ddimensions->nelts; d++) { - mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,d,mapcache_dimension*); - const char *value = apr_table_get(dimtable,dimension->name); - if(value) { - apr_table_set(req->tiles[i]->dimensions,dimension->name,value); - } - } - } - if(tileset->timedimension) { - apr_table_set(req->tiles[i]->dimensions,tileset->timedimension->key, - APR_ARRAY_IDX(timedim_selected,i,char*)); - } + req->tiles[0] = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); + if(!req->tiles[0]) { + ctx->set_error(ctx, 500, "failed to allocate tile"); + if(kvp) ctx->set_exception(ctx,"NoApplicableCode",""); + return; + } - req->tiles[i]->z = level; - req->tiles[i]->x = x; - req->tiles[i]->y = y; - if(i==0) { - /* no need to validate all the tiles as they all have the same x,y,z */ - mapcache_tileset_tile_validate(ctx,req->tiles[0]); - if(GC_HAS_ERROR(ctx)) { - if(kvp) ctx->set_exception(ctx,"TileOutOfRange",""); - return; + /*populate dimensions*/ + if(tileset->dimensions) { + int d; + for(d=0; ddimensions->nelts; d++) { + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,d,mapcache_dimension*); + const char *value = apr_table_get(dimtable,dimension->name); + if(value) { + mapcache_tile_set_requested_dimension(ctx,req->tiles[0],dimension->name,value); } } } + req->tiles[0]->z = level; + req->tiles[0]->x = x; + req->tiles[0]->y = y; + /* no need to validate all the tiles as they all have the same x,y,z */ + mapcache_tileset_tile_validate_z(ctx,req->tiles[0]); + if(GC_HAS_ERROR(ctx)) { + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileMatrix"); + return; + } + mapcache_tileset_tile_validate_x(ctx,req->tiles[0]); + if(GC_HAS_ERROR(ctx)) { + if(kvp) ctx->set_exception(ctx,"TileOutOfRange","TileCol"); + return; + } + mapcache_tileset_tile_validate_y(ctx,req->tiles[0]); + if(GC_HAS_ERROR(ctx)) { + if(kvp) ctx->set_exception(ctx,"TileOutOfRange","TileRow"); + return; + } *request = (mapcache_request*)req; return; @@ -1021,7 +976,7 @@ mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,d,mapcache_dimension*); const char *value = apr_table_get(dimtable,dimension->name); if(value) { - apr_table_set(fi->map.dimensions,dimension->name,value); + mapcache_map_set_requested_dimension(ctx,&fi->map,dimension->name,value); } } } @@ -1059,7 +1014,9 @@ "",elts[i].key,elts[i].val),NULL); } - *err_body = apr_psprintf(ctx->pool,template,msg,exceptions); + *err_body = apr_psprintf(ctx->pool,template, + mapcache_util_str_xml_escape(ctx->pool, msg, MAPCACHE_UTIL_XML_SECTION_COMMENT), + exceptions); apr_table_set(headers, "Content-Type", "application/xml"); diff -Nru mapcache-1.2.1/lib/source.c mapcache-1.6.1/lib/source.c --- mapcache-1.2.1/lib/source.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/source.c 2017-09-29 21:31:13.000000000 +0000 @@ -28,6 +28,7 @@ *****************************************************************************/ #include "mapcache.h" +#include @@ -36,6 +37,60 @@ mapcache_extent tmp_extent = {-1,-1,-1,-1}; source->data_extent = tmp_extent; source->metadata = apr_table_make(ctx->pool,3); + source->retry_count = 1; + source->retry_delay = 0.1; +} + + +void mapcache_source_render_map(mapcache_context *ctx, mapcache_source *source, mapcache_map *map) { + int i; +#ifdef DEBUG + ctx->log(ctx, MAPCACHE_DEBUG, "calling render_map on source (%s): tileset=%s, grid=%s, extent=(%f,%f,%f,%f)", + source->name, map->tileset->name, map->grid_link->grid->name, + map->extent.minx, map->extent.miny, map->extent.maxx, map->extent.maxy); +#endif + for(i=0;i<=source->retry_count;i++) { + if(i) { /* not our first try */ + ctx->log(ctx, MAPCACHE_INFO, "source (%s) render_map retry %d of %d. previous try returned error: %s", + source->name, i, source->retry_count, ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(source->retry_delay > 0) { + double wait = source->retry_delay; + int j = 0; + for(j=1;j_render_map(ctx, source, map); + if(!GC_HAS_ERROR(ctx)) + break; + } +} + +void mapcache_source_query_info(mapcache_context *ctx, mapcache_source *source, mapcache_feature_info *fi) { + int i; +#ifdef DEBUG + ctx->log(ctx, MAPCACHE_DEBUG, "calling query_info on source (%s): tileset=%s, grid=%s,", + source->name, fi->map.tileset->name, fi->map.grid_link->grid->name); +#endif + for(i=0;i<=source->retry_count;i++) { + if(i) { /* not our first try */ + ctx->log(ctx, MAPCACHE_INFO, "source (%s) render_map retry %d of %d. previous try returned error: %s", + source->name, i, source->retry_count, ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(source->retry_delay > 0) { + double wait = source->retry_delay; + int j = 0; + for(j=1;j_query_info(ctx, source, fi); + if(!GC_HAS_ERROR(ctx)) + break; + } } /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.2.1/lib/source_dummy.c mapcache-1.6.1/lib/source_dummy.c --- mapcache-1.2.1/lib/source_dummy.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/source_dummy.c 2017-09-29 21:31:13.000000000 +0000 @@ -33,11 +33,18 @@ #include #include +typedef struct mapcache_source_dummy mapcache_source_dummy; +struct mapcache_source_dummy { + mapcache_source source; + char *mapfile; + void *mapobj; +}; + /** * \private \memberof mapcache_source_dummy * \sa mapcache_source::render_map() */ -void _mapcache_source_dummy_render_map(mapcache_context *ctx, mapcache_map *map) +void _mapcache_source_dummy_render_map(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) { map->raw_image = mapcache_image_create(ctx); map->raw_image->w = map->width; @@ -48,7 +55,7 @@ apr_pool_cleanup_register(ctx->pool, map->raw_image->data,(void*)free, apr_pool_cleanup_null); } -void _mapcache_source_dummy_query(mapcache_context *ctx, mapcache_feature_info *fi) +void _mapcache_source_dummy_query(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi) { ctx->set_error(ctx,500,"dummy source does not support queries"); } @@ -57,7 +64,7 @@ * \private \memberof mapcache_source_dummy * \sa mapcache_source::configuration_parse() */ -void _mapcache_source_dummy_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source) +void _mapcache_source_dummy_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source, mapcache_cfg *config) { } @@ -79,10 +86,10 @@ } mapcache_source_init(ctx, &(source->source)); source->source.type = MAPCACHE_SOURCE_DUMMY; - source->source.render_map = _mapcache_source_dummy_render_map; + source->source._render_map = _mapcache_source_dummy_render_map; source->source.configuration_check = _mapcache_source_dummy_configuration_check; source->source.configuration_parse_xml = _mapcache_source_dummy_configuration_parse_xml; - source->source.query_info = _mapcache_source_dummy_query; + source->source._query_info = _mapcache_source_dummy_query; return (mapcache_source*)source; } diff -Nru mapcache-1.2.1/lib/source_fallback.c mapcache-1.6.1/lib/source_fallback.c --- mapcache-1.2.1/lib/source_fallback.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/source_fallback.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,165 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching support file: Mapserver Mapfile datasource + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + + +#include "mapcache.h" +#include "ezxml.h" +#include +#include + +typedef struct mapcache_source_fallback mapcache_source_fallback; +struct mapcache_source_fallback { + mapcache_source source; + apr_array_header_t *sources; +}; + +/** + * \private \memberof mapcache_source_fallback + * \sa mapcache_source::render_map() + */ +void _mapcache_source_fallback_render_map(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) +{ + mapcache_source_fallback *source = (mapcache_source_fallback*)psource; + mapcache_source *subsource; + int i; + subsource = APR_ARRAY_IDX(source->sources,0,mapcache_source*); + mapcache_source_render_map(ctx, subsource, map); + + if(GC_HAS_ERROR(ctx)) { + int first_error = ctx->get_error(ctx); + char *first_error_message = ctx->get_error_message(ctx); + ctx->log(ctx,MAPCACHE_INFO, + "failed render on primary source \"%s\" on tileset \"%s\". Falling back on secondary sources", + subsource->name,map->tileset->name); + ctx->clear_errors(ctx); + for(i=1; isources->nelts; i++) { + subsource = APR_ARRAY_IDX(source->sources,i,mapcache_source*); + mapcache_source_render_map(ctx, subsource, map); + if(GC_HAS_ERROR(ctx)) { + ctx->log(ctx,MAPCACHE_INFO, + "failed render on fallback source \"%s\" of tileset \"%s\". Continuing with other fallback sources if available", + subsource->name,map->tileset->name); + ctx->clear_errors(ctx); + continue; + } else { + return; + } + } + /* all backends failed, return primary error message */ + ctx->set_error(ctx,first_error,first_error_message); + return; + } +} + +void _mapcache_source_fallback_query(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi) +{ + mapcache_source_fallback *source = (mapcache_source_fallback*)psource; + mapcache_source *subsource; + int i; + subsource = APR_ARRAY_IDX(source->sources,0,mapcache_source*); + mapcache_source_query_info(ctx, subsource, fi); + + if(GC_HAS_ERROR(ctx)) { + int first_error = ctx->get_error(ctx); + char *first_error_message = ctx->get_error_message(ctx); + ctx->log(ctx,MAPCACHE_INFO, + "failed query_info on primary source \"%s\" on tileset \"%s\". Falling back on secondary sources", + subsource->name,fi->map.tileset->name); + ctx->clear_errors(ctx); + for(i=1; isources->nelts; i++) { + subsource = APR_ARRAY_IDX(source->sources,i,mapcache_source*); + mapcache_source_query_info(ctx, subsource, fi); + if(GC_HAS_ERROR(ctx)) { + ctx->log(ctx,MAPCACHE_INFO, + "failed query_info on fallback source \"%s\" of tileset \"%s\". Continuing with other fallback sources if available", + subsource->name,fi->map.tileset->name); + ctx->clear_errors(ctx); + continue; + } else { + return; + } + } + /* all backends failed, return primary error message */ + ctx->set_error(ctx,first_error,first_error_message); + return; + } + ctx->set_error(ctx,500,"fallback source does not support queries"); +} + +/** + * \private \memberof mapcache_source_fallback + * \sa mapcache_source::configuration_parse() + */ +void _mapcache_source_fallback_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *psource, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_source_fallback *source = (mapcache_source_fallback*)psource; + source->sources = apr_array_make(ctx->pool,3,sizeof(mapcache_source*)); + for(cur_node = ezxml_child(node,"source"); cur_node; cur_node = cur_node->next) { + mapcache_source *refsource = mapcache_configuration_get_source(config, cur_node->txt); + if(!refsource) { + ctx->set_error(ctx, 400, "fallback source \"%s\" references source \"%s\"," + " but it is not configured (hint:referenced sources must be declared before this fallback source in the xml file)", psource->name, cur_node->txt); + return; + } + APR_ARRAY_PUSH(source->sources,mapcache_source*) = refsource; + } + if(source->sources->nelts == 0) { + ctx->set_error(ctx,400,"fallback source \"%s\" does not reference any child sources", psource->name); + } +} + +/** + * \private \memberof mapcache_source_fallback + * \sa mapcache_source::configuration_check() + */ +void _mapcache_source_fallback_configuration_check(mapcache_context *ctx, mapcache_cfg *cfg, + mapcache_source *source) +{ +} + +mapcache_source* mapcache_source_fallback_create(mapcache_context *ctx) +{ + mapcache_source_fallback *source = apr_pcalloc(ctx->pool, sizeof(mapcache_source_fallback)); + if(!source) { + ctx->set_error(ctx, 500, "failed to allocate fallback source"); + return NULL; + } + mapcache_source_init(ctx, &(source->source)); + source->source.type = MAPCACHE_SOURCE_FALLBACK; + source->source._render_map = _mapcache_source_fallback_render_map; + source->source.configuration_check = _mapcache_source_fallback_configuration_check; + source->source.configuration_parse_xml = _mapcache_source_fallback_configuration_parse_xml; + source->source._query_info = _mapcache_source_fallback_query; + return (mapcache_source*)source; +} + + +/* vim: ts=2 sts=2 et sw=2 +*/ diff -Nru mapcache-1.2.1/lib/source_gdal.c mapcache-1.6.1/lib/source_gdal.c --- mapcache-1.2.1/lib/source_gdal.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/source_gdal.c 2017-09-29 21:31:13.000000000 +0000 @@ -3,10 +3,11 @@ * * Project: MapServer * Purpose: MapCache tile caching support file: GDAL datasource support (incomplete and disabled) - * Author: Thomas Bonfort and the MapServer team. + * Author: Thomas Bonfort, Even Rouault and the MapServer team. * ****************************************************************************** * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * Copyright (c) 2004, Frank Warmerdam (for GDALAutoCreateWarpedVRT who CreateWarpedVRT is derived from) * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -37,190 +38,573 @@ #include #include -#include "gdal_alg.h" +#include "gdalwarper.h" #include "cpl_string.h" #include "ogr_srs_api.h" +#include "gdal_vrt.h" -/** - * \private \memberof mapcache_source_gdal - * \sa mapcache_source::render_metatile() - */ -void _mapcache_source_gdal_render_metatile(mapcache_context *ctx, mapcache_metatile *tile) -{ - mapcache_source_gdal *gdal = (mapcache_source_gdal*)tile->tile.tileset->source; - char *srcSRS = "", *dstSRS; - mapcache_buffer *data = mapcache_buffer_create(0,ctx->pool); - GC_CHECK_ERROR(ctx); - GDALDatasetH hDataset; - - GDALAllRegister(); - OGRSpatialReferenceH hSRS; - CPLErrorReset(); +#define MAPCACHE_DEFAULT_RESAMPLE_ALG GRA_Bilinear - hSRS = OSRNewSpatialReference( NULL ); - if( OSRSetFromUserInput( hSRS, tile->tile.grid->srs ) == OGRERR_NONE ) - OSRExportToWkt( hSRS, &dstSRS ); - else { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"failed to parse gdal srs %s",tile->tile.grid->srs); - return; - } +/* Note: this will also work with GDAL >= 2.0 */ +#if GDAL_VERSION_MAJOR < 2 +#define USE_PRE_GDAL2_METHOD +#endif +/*#define USE_PRE_GDAL2_METHOD*/ - OSRDestroySpatialReference( hSRS ); +typedef struct mapcache_source_gdal mapcache_source_gdal; - hDataset = GDALOpen( gdal->datastr, GA_ReadOnly ); - if( hDataset == NULL ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"GDAL failed to open %s",gdal->datastr); +/**\class mapcache_source_gdal + * \brief GDAL mapcache_source + * \implements mapcache_source + */ +struct mapcache_source_gdal { + mapcache_source source; + const char *datastr; /**< the gdal source string*/ + const char *srs_wkt; + GDALResampleAlg eResampleAlg; /**< resampling algorithm */ + const char *srcOvrLevel; /**< strategy to pickup source overview: AUTO, NONE, + AUTO-xxx, xxxx. See -ovr doc in http://www.gdal.org/gdalwarp.html. + Only used for GDAL >= 2.0 (could probably be made to work for USE_PRE_GDAL2_METHOD with more work) */ + int bUseConnectionPool; +}; + +typedef struct { + mapcache_source_gdal *gdal; + const char *gdal_data; + const char *dst_srs; +} gdal_connection_params; + +typedef struct { + GDALDatasetH hSrcDS; + char *dst_srs_wkt; +} gdal_connection; + +void mapcache_source_gdal_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + gdal_connection_params *p = (gdal_connection_params*)params; + gdal_connection *c = malloc(sizeof(gdal_connection)); + OGRSpatialReferenceH hDstSRS; + + *conn_ = NULL; + /* -------------------------------------------------------------------- */ + /* Open source dataset. */ + /* -------------------------------------------------------------------- */ + c->hSrcDS = GDALOpen( p->gdal_data, GA_ReadOnly ); + + if( c->hSrcDS == NULL ) { + ctx->set_error(ctx, 500, "Cannot open gdal source for %s .\n", p->gdal->source.name ); + free(c); return; } /* -------------------------------------------------------------------- */ - /* Check that there's at least one raster band */ + /* Check that there's 3 or 4 raster bands. */ /* -------------------------------------------------------------------- */ - if ( GDALGetRasterCount(hDataset) == 0 ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"raster %s has no bands",gdal->datastr); + if ( GDALGetRasterCount(c->hSrcDS) != 3 && GDALGetRasterCount(c->hSrcDS) != 4) { + ctx->set_error(ctx, 500, "Input gdal source for %s has %d raster bands, but only 3 or 4 are supported.\n", + p->gdal->source.name, GDALGetRasterCount(c->hSrcDS) ); + GDALClose(c->hSrcDS); + free(c); return; } - if( GDALGetProjectionRef( hDataset ) != NULL - && strlen(GDALGetProjectionRef( hDataset )) > 0 ) - srcSRS = apr_pstrdup(ctx->pool,GDALGetProjectionRef( hDataset )); - - else if( GDALGetGCPProjection( hDataset ) != NULL - && strlen(GDALGetGCPProjection(hDataset)) > 0 - && GDALGetGCPCount( hDataset ) > 1 ) - srcSRS = apr_pstrdup(ctx->pool,GDALGetGCPProjection( hDataset )); - GDALDriverH hDriver = GDALGetDriverByName( "MEM" ); - GDALDatasetH hDstDS; - /* -------------------------------------------------------------------- */ - /* Create a transformation object from the source to */ - /* destination coordinate system. */ - /* -------------------------------------------------------------------- */ - void *hTransformArg = - GDALCreateGenImgProjTransformer( hDataset, srcSRS, - NULL, dstSRS, - TRUE, 1000.0, 0 ); - if( hTransformArg == NULL ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"gdal failed to create SRS transformation object"); + hDstSRS = OSRNewSpatialReference( NULL ); + if( OSRSetFromUserInput( hDstSRS, p->dst_srs ) == OGRERR_NONE ) { + c->dst_srs_wkt = NULL; + OSRExportToWkt( hDstSRS, &c->dst_srs_wkt ); + } + else { + ctx->set_error(ctx,500,"failed to parse gdal srs %s",p->dst_srs); + GDALClose(c->hSrcDS); + free(c); return; } - /* -------------------------------------------------------------------- */ - /* Get approximate output definition. */ - /* -------------------------------------------------------------------- */ - int nPixels, nLines; - double adfDstGeoTransform[6]; - if( GDALSuggestedWarpOutput( hDataset, - GDALGenImgProjTransform, hTransformArg, - adfDstGeoTransform, &nPixels, &nLines ) - != CE_None ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"gdal failed to create suggested warp output"); - return; + OSRDestroySpatialReference( hDstSRS ); + *conn_ = c; +} + +void mapcache_source_gdal_connection_destructor(void *conn_) { + gdal_connection *c = (gdal_connection*)conn_; + CPLFree(c->dst_srs_wkt); + GDALClose(c->hSrcDS); + free(c); +} + +static mapcache_pooled_connection* _gdal_get_connection(mapcache_context *ctx, mapcache_source_gdal *gdal, const char *dst_srs, const char *gdal_data) +{ + mapcache_pooled_connection *pc; + gdal_connection_params params; + char *key; + + params.gdal = gdal; + params.dst_srs = dst_srs; + params.gdal_data = gdal_data; + + key = apr_pstrcat(ctx->pool, gdal_data, dst_srs, NULL); + + pc = mapcache_connection_pool_get_connection(ctx,key,mapcache_source_gdal_connection_constructor, + mapcache_source_gdal_connection_destructor, ¶ms); + return pc; +} +#ifdef USE_PRE_GDAL2_METHOD +/* Creates a (virtual) dataset that matches an overview level of the source + dataset. This dataset has references on the source dataset, so it should + be closed before it. +*/ +static GDALDatasetH CreateOverviewVRTDataset(GDALDatasetH hSrcDS, + int iOvrLevel) +{ + double adfSrcGeoTransform[6]; + int iBand; + GDALRasterBandH hFirstBand = GDALGetRasterBand(hSrcDS, 1); + GDALRasterBandH hOvr = GDALGetOverview( hFirstBand, iOvrLevel ); + int nFullResXSize = GDALGetRasterBandXSize(hFirstBand); + int nFullResYSize = GDALGetRasterBandYSize(hFirstBand); + int nVRTXSize = GDALGetRasterBandXSize(hOvr); + int nVRTYSize = GDALGetRasterBandYSize(hOvr); + + VRTDatasetH hVRTDS = VRTCreate( nVRTXSize, nVRTYSize ); + + /* Scale geotransform */ + if( GDALGetGeoTransform( hSrcDS, adfSrcGeoTransform) == CE_None ) + { + adfSrcGeoTransform[1] *= (double)nFullResXSize / nVRTXSize; + adfSrcGeoTransform[2] *= (double)nFullResYSize / nVRTYSize; + adfSrcGeoTransform[4] *= (double)nFullResXSize / nVRTXSize; + adfSrcGeoTransform[5] *= (double)nFullResYSize / nVRTYSize; + GDALSetProjection( hVRTDS, GDALGetProjectionRef(hSrcDS) ); + GDALSetGeoTransform( hVRTDS, adfSrcGeoTransform ); + } + + /* Scale GCPs */ + if( GDALGetGCPCount(hSrcDS) > 0 ) + { + const GDAL_GCP* pasGCPsMain = GDALGetGCPs(hSrcDS); + int nGCPCount = GDALGetGCPCount(hSrcDS); + GDAL_GCP* pasGCPList = GDALDuplicateGCPs( nGCPCount, pasGCPsMain ); + int i; + for(i = 0; i < nGCPCount; i++) + { + pasGCPList[i].dfGCPPixel *= (double)nVRTXSize / nFullResXSize; + pasGCPList[i].dfGCPLine *= (double)nVRTYSize / nFullResYSize; + } + GDALSetGCPs( hVRTDS, nGCPCount, pasGCPList, GDALGetGCPProjection(hSrcDS) ); + GDALDeinitGCPs( nGCPCount, pasGCPList ); + CPLFree( pasGCPList ); + } + + /* Create bands */ + for(iBand = 1; iBand <= GDALGetRasterCount(hSrcDS); iBand++ ) + { + GDALRasterBandH hSrcBand; + GDALRasterBandH hVRTBand; + int bNoDataSet = FALSE; + double dfNoData; + + VRTAddBand( hVRTDS, GDT_Byte, NULL ); + hVRTBand = GDALGetRasterBand(hVRTDS, iBand); + hSrcBand = GDALGetOverview(GDALGetRasterBand(hSrcDS, iBand), iOvrLevel); + dfNoData = GDALGetRasterNoDataValue(hSrcBand, &bNoDataSet); + if( bNoDataSet ) + GDALSetRasterNoDataValue(hVRTBand, dfNoData); + + /* Note: the consumer of this VRT is the warper, which doesn't do any */ + /* subsampled RasterIO requests, so NEAR is fine */ + VRTAddSimpleSource( hVRTBand, hSrcBand, + 0, 0, nVRTXSize, nVRTYSize, + 0, 0, nVRTXSize, nVRTYSize, + "NEAR", VRT_NODATA_UNSET ); } - GDALDestroyGenImgProjTransformer( hTransformArg ); - double dfXRes = (tile->bbox[2] - tile->bbox[0]) / tile->sx; - double dfYRes = (tile->bbox[3] - tile->bbox[1]) / tile->sy; - - adfDstGeoTransform[0] = tile->bbox[0]; - adfDstGeoTransform[3] = tile->bbox[3]; - adfDstGeoTransform[1] = dfXRes; - adfDstGeoTransform[5] = -dfYRes; - hDstDS = GDALCreate( hDriver, "tempd_gdal_image", tile->sx, tile->sy, 4, GDT_Byte, NULL ); + return hVRTDS; +} +#endif + +/* Derived from GDALAutoCreateWarpedVRT(), with various improvements. */ +/* Returns a warped VRT that covers the passed extent, in pszDstWKT. */ +/* The provided width and height are used, but the size of the returned dataset */ +/* may not match those values. In the USE_PRE_GDAL2_METHOD, it should match them. */ +/* In the non USE_PRE_GDAL2_METHOD case, it might be a multiple of those values. */ +/* phTmpDS is an output parameter (a temporary VRT in the USE_PRE_GDAL2_METHOD case). */ +static GDALDatasetH +CreateWarpedVRT( GDALDatasetH hSrcDS, + const char *pszSrcWKT, + const char *pszDstWKT, + int width, int height, + const mapcache_extent *extent, + GDALResampleAlg eResampleAlg, + double dfMaxError, + char** papszWarpOptions, + GDALDatasetH *phTmpDS ) - /* -------------------------------------------------------------------- */ - /* Write out the projection definition. */ - /* -------------------------------------------------------------------- */ - GDALSetProjection( hDstDS, dstSRS ); - GDALSetGeoTransform( hDstDS, adfDstGeoTransform ); - char **papszWarpOptions = NULL; - papszWarpOptions = CSLSetNameValue( papszWarpOptions, "INIT", "0" ); +{ + int i; + GDALWarpOptions *psWO; + double adfDstGeoTransform[6]; + GDALDatasetH hDstDS; + int nDstPixels, nDstLines; + CPLErr eErr; + int bHaveNodata = FALSE; + char** papszOptions = NULL; + +/* -------------------------------------------------------------------- */ +/* Populate the warp options. */ +/* -------------------------------------------------------------------- */ + + psWO = GDALCreateWarpOptions(); + psWO->papszWarpOptions = CSLDuplicate(papszWarpOptions); + + psWO->eResampleAlg = eResampleAlg; + + psWO->hSrcDS = hSrcDS; + + psWO->nBandCount = GDALGetRasterCount( hSrcDS ); + if( psWO->nBandCount == 4 ) + { + psWO->nBandCount = 3; + psWO->nSrcAlphaBand = 4; + psWO->nDstAlphaBand = 4; + } + /* Due to the reprojection, we might get transparency in the edges */ + else if( psWO->nBandCount == 3 ) + psWO->nDstAlphaBand = 4; + + psWO->panSrcBands = (int*)CPLMalloc( sizeof(int) * psWO->nBandCount ); + psWO->panDstBands = (int*)CPLMalloc( sizeof(int) * psWO->nBandCount ); + + for( i = 0; i < psWO->nBandCount; i++ ) + { + psWO->panSrcBands[i] = i+1; + psWO->panDstBands[i] = i+1; + } + +/* -------------------------------------------------------------------- */ +/* Set nodata values if existing */ +/* -------------------------------------------------------------------- */ + GDALGetRasterNoDataValue( GDALGetRasterBand(hSrcDS, 1), &bHaveNodata); + if( bHaveNodata ) + { + psWO->padfSrcNoDataReal = (double *) + CPLMalloc(psWO->nBandCount*sizeof(double)); + psWO->padfSrcNoDataImag = (double *) + CPLCalloc(psWO->nBandCount, sizeof(double)); /* zero initialized */ + + for( i = 0; i < psWO->nBandCount; i++ ) + { + GDALRasterBandH hBand = GDALGetRasterBand( hSrcDS, i+1 ); + + double dfReal = GDALGetRasterNoDataValue( hBand, &bHaveNodata ); + + if( bHaveNodata ) + { + psWO->padfSrcNoDataReal[i] = dfReal; + } + else + { + psWO->padfSrcNoDataReal[i] = -123456.789; + } + } + } +/* -------------------------------------------------------------------- */ +/* Create the transformer. */ +/* -------------------------------------------------------------------- */ + psWO->pfnTransformer = GDALGenImgProjTransform; + psWO->pTransformerArg = + GDALCreateGenImgProjTransformer( psWO->hSrcDS, pszSrcWKT, + NULL, pszDstWKT, + TRUE, 1.0, 0 ); + + if( psWO->pTransformerArg == NULL ) + { + GDALDestroyWarpOptions( psWO ); + return NULL; + } +/* -------------------------------------------------------------------- */ +/* Figure out the desired output bounds and resolution. */ +/* -------------------------------------------------------------------- */ + eErr = + GDALSuggestedWarpOutput( hSrcDS, psWO->pfnTransformer, + psWO->pTransformerArg, + adfDstGeoTransform, &nDstPixels, &nDstLines ); + if( eErr != CE_None ) + { + GDALDestroyTransformer( psWO->pTransformerArg ); + GDALDestroyWarpOptions( psWO ); + return NULL; + } - /* -------------------------------------------------------------------- */ - /* Create a transformation object from the source to */ - /* destination coordinate system. */ - /* -------------------------------------------------------------------- */ - GDALTransformerFunc pfnTransformer = NULL; - void *hGenImgProjArg=NULL, *hApproxArg=NULL; - hTransformArg = hGenImgProjArg = - GDALCreateGenImgProjTransformer( hDataset, srcSRS, - hDstDS, dstSRS, - TRUE, 1000.0, 0 ); - - if( hTransformArg == NULL ) - exit( 1 ); - - pfnTransformer = GDALGenImgProjTransform; - - hTransformArg = hApproxArg = - GDALCreateApproxTransformer( GDALGenImgProjTransform, - hGenImgProjArg, 0.125 ); - pfnTransformer = GDALApproxTransform; +/* -------------------------------------------------------------------- */ +/* To minimize the risk of extra resampling done by generic */ +/* RasterIO itself and maximize resampling done in the wraper, */ +/* adjust the resolution so that the overview factor of the output */ +/* dataset that will indirectly query matches an exiting overview */ +/* factor of the input dataset. */ +/* -------------------------------------------------------------------- */ + { + double dfDesiredXRes = (extent->maxx - extent->minx) / width; + double dfDesiredYRes = (extent->maxy - extent->miny) / height; + double dfDesiredRes = MIN( dfDesiredXRes, dfDesiredYRes ); + double dfGuessedFullRes = MIN( adfDstGeoTransform[1], + fabs(adfDstGeoTransform[5]) ); + double dfApproxDstOvrRatio = dfDesiredRes / dfGuessedFullRes; + + GDALRasterBandH hFirstBand = GDALGetRasterBand(hSrcDS, 1); + int nOvrCount = GDALGetOverviewCount(hFirstBand); + int nSrcXSize = GDALGetRasterBandXSize(hFirstBand); + int i; + double dfSrcOvrRatio = 1.0; +#ifdef USE_PRE_GDAL2_METHOD + int iSelectedOvr = -1; +#endif + for( i = 0; *phTmpDS == NULL && i < nOvrCount; i ++) + { + GDALRasterBandH hOvr = GDALGetOverview(hFirstBand, i); + int nOvrXSize = GDALGetRasterBandXSize(hOvr); + double dfCurOvrRatio = (double)nSrcXSize / nOvrXSize; + if(dfCurOvrRatio > dfApproxDstOvrRatio+0.1 ) /* +0.1 to avoid rounding issues */ + { + break; + } + dfSrcOvrRatio = dfCurOvrRatio; +#ifdef USE_PRE_GDAL2_METHOD + iSelectedOvr = i; +#endif + } + +#ifdef USE_PRE_GDAL2_METHOD + if( iSelectedOvr >= 0 ) + { + GDALDestroyTransformer( psWO->pTransformerArg ); + GDALDestroyWarpOptions( psWO ); + + *phTmpDS = CreateOverviewVRTDataset(hSrcDS, iSelectedOvr); + return CreateWarpedVRT( *phTmpDS, + pszSrcWKT, + pszDstWKT, + width, height, + extent, + eResampleAlg, + dfMaxError, + papszWarpOptions, + phTmpDS ); + } +#endif - /* -------------------------------------------------------------------- */ - /* Now actually invoke the warper to do the work. */ - /* -------------------------------------------------------------------- */ - GDALSimpleImageWarp( hDataset, hDstDS, 0, NULL, - pfnTransformer, hTransformArg, - GDALDummyProgress, NULL, papszWarpOptions ); + adfDstGeoTransform[1] = dfDesiredXRes / dfSrcOvrRatio; + adfDstGeoTransform[5] = -dfDesiredYRes / dfSrcOvrRatio; + } - CSLDestroy( papszWarpOptions ); +/* -------------------------------------------------------------------- */ +/* Compute geotransform and raster dimension for our extent of */ +/* interest. */ +/* -------------------------------------------------------------------- */ + adfDstGeoTransform[0] = extent->minx; + adfDstGeoTransform[2] = 0.0; + adfDstGeoTransform[3] = extent->maxy; + adfDstGeoTransform[4] = 0.0; + nDstPixels = (int)( (extent->maxx - extent->minx) / adfDstGeoTransform[1] + 0.5 ); + nDstLines = (int)( (extent->maxy - extent->miny) / fabs(adfDstGeoTransform[5]) + 0.5 ); + /*printf("nDstPixels=%d nDstLines=%d\n", nDstPixels, nDstLines);*/ + +/* -------------------------------------------------------------------- */ +/* Update the transformer to include an output geotransform */ +/* back to pixel/line coordinates. */ +/* -------------------------------------------------------------------- */ + + GDALSetGenImgProjTransformerDstGeoTransform( + psWO->pTransformerArg, adfDstGeoTransform ); + +/* -------------------------------------------------------------------- */ +/* Do we want to apply an approximating transformation? */ +/* -------------------------------------------------------------------- */ + if( dfMaxError > 0.0 ) + { + psWO->pTransformerArg = + GDALCreateApproxTransformer( psWO->pfnTransformer, + psWO->pTransformerArg, + dfMaxError ); + psWO->pfnTransformer = GDALApproxTransform; + GDALApproxTransformerOwnsSubtransformer(psWO->pTransformerArg, TRUE); + } - if( hApproxArg != NULL ) - GDALDestroyApproxTransformer( hApproxArg ); +/* -------------------------------------------------------------------- */ +/* Create the VRT file. */ +/* -------------------------------------------------------------------- */ + + /* We could potentially used GDALCreateWarpedVRT() instead of this logic */ + /* but GDALCreateWarpedVRT() in GDAL < 2.0.1 doesn't create the destination */ + /* alpha band. */ + + papszOptions = CSLSetNameValue(NULL, "SUBCLASS", "VRTWarpedDataset"); + hDstDS = GDALCreate( GDALGetDriverByName("VRT"), "", nDstPixels, nDstLines, + psWO->nBandCount + ((psWO->nDstAlphaBand != 0) ? 1 : 0), + GDT_Byte, papszOptions ); + CSLDestroy(papszOptions); + if( hDstDS == NULL ) + { + GDALDestroyWarpOptions( psWO ); + return NULL; + } - if( hGenImgProjArg != NULL ) - GDALDestroyGenImgProjTransformer( hGenImgProjArg ); + psWO->hDstDS = hDstDS; - if(GDALGetRasterCount(hDstDS) != 4) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"gdal did not create a 4 band image"); - return; + GDALSetGeoTransform( hDstDS, adfDstGeoTransform ); + if( GDALInitializeWarpedVRT( hDstDS, psWO ) != CE_None ) + { + GDALClose(hDstDS); + GDALDestroyWarpOptions( psWO ); + return NULL; + } + + GDALDestroyWarpOptions( psWO ); + + return hDstDS; +} + + +/** + * \private \memberof mapcache_source_gdal + * \sa mapcache_source::render_metatile() + */ +void _mapcache_source_gdal_render_metatile(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) +{ + mapcache_source_gdal *gdal = (mapcache_source_gdal*)psource; + gdal_connection *gdal_conn; + GDALDatasetH hDstDS; + GDALDatasetH hTmpDS = NULL; + mapcache_buffer *data; + unsigned char *rasterdata; + CPLErr eErr; + mapcache_pooled_connection *pc = NULL; + int bands_bgra[] = { 3, 2, 1, 4 }; /* mapcache buffer order is BGRA */ + + CPLErrorReset(); + + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + pc = _gdal_get_connection(ctx, gdal, map->grid_link->grid->srs, gdal->datastr ); + GC_CHECK_ERROR(ctx); + gdal_conn = (gdal_connection*) pc->connection; + } else { + gdal_connection_params params; + params.gdal = gdal; + params.dst_srs = map->grid_link->grid->srs; + params.gdal_data = gdal->datastr; + mapcache_source_gdal_connection_constructor(ctx, (void**)(&gdal_conn), ¶ms); + GC_CHECK_ERROR(ctx); } - GDALRasterBandH *redband, *greenband, *blueband, *alphaband; - redband = GDALGetRasterBand(hDstDS,1); - greenband = GDALGetRasterBand(hDstDS,2); - blueband = GDALGetRasterBand(hDstDS,3); - alphaband = GDALGetRasterBand(hDstDS,4); - - unsigned char *rasterdata = apr_palloc(ctx->pool,tile->sx*tile->sy*4); - data->buf = rasterdata; - data->avail = tile->sx*tile->sy*4; - data->size = tile->sx*tile->sy*4; - - GDALRasterIO(redband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - GDALRasterIO(greenband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata+1),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - GDALRasterIO(blueband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata+2),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - if(GDALGetRasterCount(hDataset)==4) - GDALRasterIO(alphaband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata+3),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - else { - unsigned char *alphaptr; - int i; - for(alphaptr = rasterdata+3, i=0; isx*tile->sy; i++, alphaptr+=4) { - *alphaptr = 255; + + hDstDS = CreateWarpedVRT( gdal_conn->hSrcDS, gdal->srs_wkt, gdal_conn->dst_srs_wkt, + map->width, map->height, + &map->extent, + gdal->eResampleAlg, 0.125, NULL, &hTmpDS ); + + if( hDstDS == NULL ) { + ctx->set_error(ctx, 500,"CreateWarpedVRT() failed"); + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); + } + return; + } + + if(GDALGetRasterCount(hDstDS) != 4) { + ctx->set_error(ctx, 500,"gdal did not create a 4 band image"); + GDALClose(hDstDS); /* close first this one, as it references hSrcDS */ + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); } + return; } - tile->imdata = mapcache_image_create(ctx); - tile->imdata->w = tile->sx; - tile->imdata->h = tile->sy; - tile->imdata->stride = tile->sx * 4; - tile->imdata->data = rasterdata; + data = mapcache_buffer_create(map->height*map->width*4,ctx->pool); + rasterdata = data->buf; + +#if GDAL_VERSION_MAJOR >= 2 + { + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if( gdal->eResampleAlg == GRA_Bilinear ) + sExtraArg.eResampleAlg = GRIORA_Bilinear; + else if( gdal->eResampleAlg == GRA_Cubic ) + sExtraArg.eResampleAlg = GRIORA_Cubic; + else if( gdal->eResampleAlg == GRA_CubicSpline ) + sExtraArg.eResampleAlg = GRIORA_CubicSpline; + else if( gdal->eResampleAlg == GRA_Lanczos ) + sExtraArg.eResampleAlg = GRIORA_Lanczos; + else if( gdal->eResampleAlg == GRA_Average ) + sExtraArg.eResampleAlg = GRIORA_Average; + + if( gdal->srcOvrLevel != NULL ) + { + /* If the user specified a particular strategy to choose the source */ + /* overview level, apply it now */ + GDALSetMetadataItem(hDstDS, "SrcOvrLevel",gdal->srcOvrLevel, NULL); + } + /* Hopefully, given how we adjust hDstDS resolution, we should query */ + /* exactly at the resolution of one overview level of hDstDS, and not */ + /* do extra resampling in generic RasterIO, but just in case specify */ + /* the resampling alg in sExtraArg. */ + eErr = GDALDatasetRasterIOEx( hDstDS, GF_Read,0,0, + GDALGetRasterXSize(hDstDS), + GDALGetRasterYSize(hDstDS), + rasterdata,map->width,map->height,GDT_Byte, + 4, bands_bgra, + 4,4*map->width,1, &sExtraArg ); + } +#else + eErr = GDALDatasetRasterIO( hDstDS, GF_Read,0,0, + GDALGetRasterXSize(hDstDS), + GDALGetRasterYSize(hDstDS), + rasterdata,map->width,map->height,GDT_Byte, + 4, bands_bgra, + 4,4*map->width,1 ); +#endif + + if( eErr != CE_None ) { + ctx->set_error(ctx, 500,"GDAL I/O error occurred"); + GDALClose(hDstDS); /* close first this one, as it references hTmpDS or hSrcDS */ + if( hTmpDS ) + GDALClose(hTmpDS); /* references hSrcDS, so close before */ + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); + } + return; + } - GDALClose( hDstDS ); - GDALClose( hDataset); + map->raw_image = mapcache_image_create(ctx); + map->raw_image->w = map->width; + map->raw_image->h = map->height; + map->raw_image->stride = map->width * 4; + map->raw_image->data = rasterdata; + map->raw_image->has_alpha = MC_ALPHA_UNKNOWN; + + GDALClose( hDstDS ); /* close first this one, as it references hTmpDS or hSrcDS */ + if( hTmpDS ) + GDALClose(hTmpDS); /* references hSrcDS, so close before */ + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_release_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); + } } /** * \private \memberof mapcache_source_gdal * \sa mapcache_source::configuration_parse() */ -void _mapcache_source_gdal_configuration_parse(mapcache_context *ctx, ezxml_t node, mapcache_source *source) +void _mapcache_source_gdal_configuration_parse(mapcache_context *ctx, ezxml_t node, mapcache_source *source, mapcache_cfg *config) { ezxml_t cur_node; mapcache_source_gdal *src = (mapcache_source_gdal*)source; @@ -228,12 +612,41 @@ if ((cur_node = ezxml_child(node,"data")) != NULL) { src->datastr = apr_pstrdup(ctx->pool,cur_node->txt); } + if ((cur_node = ezxml_child(node,"connection_pooled")) != NULL) { + if(!strcasecmp(cur_node->txt,"false")) { + src->bUseConnectionPool = MAPCACHE_FALSE; + } else if(!strcasecmp(cur_node->txt,"true")) { + src->bUseConnectionPool = MAPCACHE_TRUE; + } else { + ctx->set_error(ctx,400,"failed to parse (%s). Expecting true or false",cur_node->txt); + return; + } + } - if ((cur_node = ezxml_child(node,"gdalparams")) != NULL) { - for(cur_node = cur_node->child; cur_node; cur_node = cur_node->sibling) { - apr_table_set(src->gdal_params, cur_node->name, cur_node->txt); + if ((cur_node = ezxml_child(node,"resample")) != NULL && *cur_node->txt) { + if( EQUALN( cur_node->txt, "NEAR", 4) ) + src->eResampleAlg = GRA_NearestNeighbour; + else if( EQUAL( cur_node->txt, "BILINEAR") ) + src->eResampleAlg = GRA_Bilinear; + else if( EQUAL( cur_node->txt, "CUBIC") ) + src->eResampleAlg = GRA_Cubic; + else if( EQUAL( cur_node->txt, "CUBICSPLINE") ) + src->eResampleAlg = GRA_CubicSpline; + else if( EQUAL( cur_node->txt, "LANCZOS") ) + src->eResampleAlg = GRA_Lanczos; +#if GDAL_VERSION_MAJOR >= 2 + else if( EQUAL( cur_node->txt, "AVERAGE") ) + src->eResampleAlg = GRA_Average; +#endif + else { + ctx->set_error(ctx, 500, "unsupported gdal : %s", cur_node->txt); + return; } } + + if ((cur_node = ezxml_child(node,"overview-strategy")) != NULL && *cur_node->txt) { + src->srcOvrLevel = apr_pstrdup(ctx->pool,cur_node->txt); + } } /** @@ -244,16 +657,31 @@ mapcache_source *source) { mapcache_source_gdal *src = (mapcache_source_gdal*)source; + GDALDatasetH hDataset; + /* check all required parameters are configured */ - if(!strlen(src->datastr)) { - ctx->set_error(ctx, MAPCACHE_SOURCE_GDAL_ERROR, "gdal source %s has no data",source->name); + if( src->datastr == NULL || !strlen(src->datastr)) { + ctx->set_error(ctx, 500, "gdal source %s has no data",source->name); + return; + } + hDataset = GDALOpen(src->datastr,GA_ReadOnly); + if( hDataset == NULL ) { + ctx->set_error(ctx, 500, "gdalOpen failed on data %s", src->datastr); return; } - src->poDataset = (GDALDatasetH*)GDALOpen(src->datastr,GA_ReadOnly); - if( src->poDataset == NULL ) { - ctx->set_error(ctx, MAPCACHE_SOURCE_GDAL_ERROR, "gdalOpen failed on data %s", src->datastr); + if( GDALGetProjectionRef( hDataset ) != NULL + && strlen(GDALGetProjectionRef( hDataset )) > 0 ) { + src->srs_wkt = apr_pstrdup(ctx->pool,GDALGetProjectionRef( hDataset )); + } else if( GDALGetGCPProjection( hDataset ) != NULL + && strlen(GDALGetGCPProjection(hDataset)) > 0 + && GDALGetGCPCount( hDataset ) > 1 ) { + src->srs_wkt = apr_pstrdup(ctx->pool,GDALGetGCPProjection( hDataset )); + } else { + ctx->set_error(ctx, 500, "Input gdal source for %s has no defined SRS\n", source->name ); + GDALClose(hDataset); return; } + GDALClose(hDataset); } #endif //USE_GDAL @@ -261,18 +689,19 @@ mapcache_source* mapcache_source_gdal_create(mapcache_context *ctx) { #ifdef USE_GDAL - GDALAllRegister(); mapcache_source_gdal *source = apr_pcalloc(ctx->pool, sizeof(mapcache_source_gdal)); if(!source) { - ctx->set_error(ctx, MAPCACHE_ALLOC_ERROR, "failed to allocate gdal source"); + ctx->set_error(ctx, 500, "failed to allocate gdal source"); return NULL; } mapcache_source_init(ctx, &(source->source)); source->source.type = MAPCACHE_SOURCE_GDAL; - source->source.render_metatile = _mapcache_source_gdal_render_metatile; + source->source._render_map = _mapcache_source_gdal_render_metatile; source->source.configuration_check = _mapcache_source_gdal_configuration_check; - source->source.configuration_parse = _mapcache_source_gdal_configuration_parse; - source->gdal_params = apr_table_make(ctx->pool,4); + source->source.configuration_parse_xml = _mapcache_source_gdal_configuration_parse; + source->eResampleAlg = MAPCACHE_DEFAULT_RESAMPLE_ALG; + source->bUseConnectionPool = MAPCACHE_TRUE; + GDALAllRegister(); return (mapcache_source*)source; #else ctx->set_error(ctx, 400, "failed to create gdal source, GDAL support is not compiled in this version"); diff -Nru mapcache-1.2.1/lib/source_mapserver.c mapcache-1.6.1/lib/source_mapserver.c --- mapcache-1.2.1/lib/source_mapserver.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/source_mapserver.c 2017-09-29 21:31:13.000000000 +0000 @@ -41,8 +41,15 @@ #include #include -/* hash table key = source->name, value = apr_reslist_t of mapObjs */ -static apr_hash_t *mapobj_container = NULL; +/**\class mapcache_source_mapserver + * \brief WMS mapcache_source + * \implements mapcache_source + */ +typedef struct mapcache_source_mapserver mapcache_source_mapserver; +struct mapcache_source_mapserver { + mapcache_source source; + char *mapfile; +}; struct mc_mapobj { mapObj *map; @@ -50,91 +57,44 @@ char *error; }; -static apr_status_t _ms_get_mapobj(void **conn_, void *params, apr_pool_t *pool) -{ +void mapcache_mapserver_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { mapcache_source_mapserver *src = (mapcache_source_mapserver*) params; struct mc_mapobj *mcmap = calloc(1,sizeof(struct mc_mapobj)); - *conn_ = mcmap; mcmap->map = msLoadMap(src->mapfile,NULL); if(!mcmap->map) { errorObj *errors = NULL; - msWriteError(stderr); + ctx->set_error(ctx, 500, "Failed to load mapfile '%s'",src->mapfile); errors = msGetErrorObj(); - mcmap->error = apr_psprintf(pool,"Failed to load mapfile '%s'. Mapserver reports: %s",src->mapfile, errors->message); - return APR_EGENERAL; + while(errors) { + ctx->set_error(ctx, 500, "Failed to load mapfile '%s'. Mapserver reports: %s",src->mapfile, errors->message); + errors = errors->next; + } + return; } msMapSetLayerProjections(mcmap->map); - return APR_SUCCESS; + *conn_ = mcmap; } -static apr_status_t _ms_free_mapobj(void *conn_, void *params, apr_pool_t *pool) -{ +void mapcache_mapserver_connection_destructor(void *conn_) { struct mc_mapobj *mcmap = (struct mc_mapobj*) conn_; msFreeMap(mcmap->map); free(mcmap); - return APR_SUCCESS; -} - -static struct mc_mapobj* _get_mapboj(mapcache_context *ctx, mapcache_map *map) { - apr_status_t rv; - mapcache_source_mapserver *src = (mapcache_source_mapserver*) map->tileset->source; - struct mc_mapobj *mcmap; - apr_reslist_t *mapobjs = NULL; - if(!mapobj_container || NULL == (mapobjs = apr_hash_get(mapobj_container,src->source.name,APR_HASH_KEY_STRING))) { -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(!mapobj_container) { - mapobj_container = apr_hash_make(ctx->process_pool); - } - mapobjs = apr_hash_get(mapobj_container,src->source.name,APR_HASH_KEY_STRING); - if(!mapobjs) { - apr_status_t rv; - rv = apr_reslist_create(&mapobjs, - 0 /* min */, - 1 /* soft max */, - 30 /* hard max */, - 6 * 1000000 /*6 seconds, ttl*/, - _ms_get_mapobj, /* resource constructor */ - _ms_free_mapobj, /* resource destructor */ - src, ctx->process_pool); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to create mapobj connection pool for cache %s", src->source.name); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(mapobj_container,src->source.name,APR_HASH_KEY_STRING,mapobjs); - } - assert(mapobjs); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - } - rv = apr_reslist_acquire(mapobjs, (void **) &mcmap); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to aquire mappObj instance: %s", mcmap->error); - return NULL; - } - return mcmap; } -static void _release_mapboj(mapcache_context *ctx, mapcache_map *map, struct mc_mapobj *mcmap) +static mapcache_pooled_connection* _mapserver_get_connection(mapcache_context *ctx, mapcache_map *map) { - mapcache_source_mapserver *src = (mapcache_source_mapserver*) map->tileset->source; - msFreeLabelCache(&mcmap->map->labelcache); - apr_reslist_t *mapobjs = apr_hash_get(mapobj_container,src->source.name, APR_HASH_KEY_STRING); - assert(mapobjs); - if (GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(mapobjs, (void*) mcmap); - } else { - apr_reslist_release(mapobjs, (void*) mcmap); + mapcache_pooled_connection *pc; + char *key = apr_psprintf(ctx->pool, "ms_src_%s", map->tileset->source->name); + + pc = mapcache_connection_pool_get_connection(ctx, key, mapcache_mapserver_connection_constructor, + mapcache_mapserver_connection_destructor, map->tileset->source); + if(!GC_HAS_ERROR(ctx) && pc && pc->connection) { } + + return pc; } + + /** * \private \memberof mapcache_source_mapserver * \sa mapcache_source::render_map() @@ -142,15 +102,27 @@ void _mapcache_source_mapserver_render_map(mapcache_context *ctx, mapcache_map *map) { errorObj *errors = NULL; + mapcache_pooled_connection *pc; + struct mc_mapobj *mcmap; + double dx, dy; + rasterBufferObj rb; + imageObj *image; - struct mc_mapobj *mcmap = _get_mapboj(ctx,map); + pc = _mapserver_get_connection(ctx, map); + GC_CHECK_ERROR(ctx); + + mcmap = pc->connection; GC_CHECK_ERROR(ctx); if(mcmap->grid_link != map->grid_link) { if (msLoadProjectionString(&(mcmap->map->projection), map->grid_link->grid->srs) != 0) { + ctx->set_error(ctx,500, "Unable to set projection on mapObj."); errors = msGetErrorObj(); - ctx->set_error(ctx,500, "Unable to set projection on mapObj. MapServer reports: %s", errors->message); - _release_mapboj(ctx,map,mcmap); + while(errors) { + ctx->set_error(ctx,500, "Unable to set projection on mapObj. MapServer reports: %s", errors->message); + errors = errors->next; + } + mapcache_connection_pool_invalidate_connection(ctx,pc); return; } switch(map->grid_link->grid->unit) { @@ -173,7 +145,6 @@ ** pixel to center of pixel. Here we try to adjust the WMS extents ** in by half a pixel. */ - double dx, dy; dx = (map->extent.maxx - map->extent.minx) / (map->width*2); dy = (map->extent.maxy - map->extent.miny) / (map->height*2); @@ -183,20 +154,27 @@ mcmap->map->extent.maxy = map->extent.maxy - dy; msMapSetSize(mcmap->map, map->width, map->height); - imageObj *image = msDrawMap(mcmap->map, MS_FALSE); + image = msDrawMap(mcmap->map, MS_FALSE); if(!image) { + ctx->set_error(ctx,500, "MapServer failed to create image."); errors = msGetErrorObj(); - ctx->set_error(ctx,500, "MapServer failed to create image. MapServer reports: %s", errors->message); - _release_mapboj(ctx,map,mcmap); + while(errors) { + ctx->set_error(ctx,500, "MapServer reports: %s", errors->message); + errors = errors->next; + } + mapcache_connection_pool_invalidate_connection(ctx,pc); return; } - rasterBufferObj rb; if(image->format->vtable->supports_pixel_buffer) { - image->format->vtable->getRasterBufferHandle(image,&rb); + if( MS_SUCCESS != image->format->vtable->getRasterBufferHandle(image,&rb)) { + ctx->set_error(ctx,500,"failed to get mapserver raster buffer handle"); + mapcache_connection_pool_invalidate_connection(ctx,pc); + return; + } } else { ctx->set_error(ctx,500,"format %s has no pixel export",image->format->name); - _release_mapboj(ctx,map,mcmap); + mapcache_connection_pool_invalidate_connection(ctx,pc); return; } @@ -208,11 +186,11 @@ memcpy(map->raw_image->data,rb.data.rgba.pixels,map->width*map->height*4); apr_pool_cleanup_register(ctx->pool, map->raw_image->data,(void*)free, apr_pool_cleanup_null); msFreeImage(image); - _release_mapboj(ctx,map,mcmap); + mapcache_connection_pool_release_connection(ctx,pc); } -void _mapcache_source_mapserver_query(mapcache_context *ctx, mapcache_feature_info *fi) +void _mapcache_source_mapserver_query(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi) { ctx->set_error(ctx,500,"mapserver source does not support queries"); } @@ -238,6 +216,7 @@ mapcache_source *source) { mapcache_source_mapserver *src = (mapcache_source_mapserver*)source; + mapObj *map; /* check all required parameters are configured */ if(!src->mapfile) { ctx->set_error(ctx, 400, "mapserver source %s has no configured",source->name); @@ -250,7 +229,7 @@ msSetup(); /* do a test load to check the mapfile is correct */ - mapObj *map = msLoadMap(src->mapfile, NULL); + map = msLoadMap(src->mapfile, NULL); if(!map) { msWriteError(stderr); ctx->set_error(ctx,400,"failed to load mapfile \"%s\"",src->mapfile); @@ -278,6 +257,7 @@ mapcache_source* mapcache_source_mapserver_create(mapcache_context *ctx) { ctx->set_error(ctx, 500, "mapserver source not configured for this build"); + return NULL; } #endif diff -Nru mapcache-1.2.1/lib/source_wms.c mapcache-1.6.1/lib/source_wms.c --- mapcache-1.2.1/lib/source_wms.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/source_wms.c 2017-09-29 21:31:13.000000000 +0000 @@ -33,13 +33,28 @@ #include #include +typedef struct mapcache_source_wms mapcache_source_wms; + +/**\class mapcache_source_wms + * \brief WMS mapcache_source + * \implements mapcache_source + */ +struct mapcache_source_wms { + mapcache_source source; + apr_table_t *wms_default_params; /**< default WMS parameters (SERVICE,REQUEST,STYLES,VERSION) */ + apr_table_t *getmap_params; /**< WMS parameters specified in configuration */ + apr_table_t *getfeatureinfo_params; /**< WMS parameters specified in configuration */ + mapcache_http *http; +}; + /** * \private \memberof mapcache_source_wms * \sa mapcache_source::render_map() */ -void _mapcache_source_wms_render_map(mapcache_context *ctx, mapcache_map *map) +void _mapcache_source_wms_render_map(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) { - mapcache_source_wms *wms = (mapcache_source_wms*)map->tileset->source; + mapcache_source_wms *wms = (mapcache_source_wms*)psource; + mapcache_http *http; apr_table_t *params = apr_table_clone(ctx->pool,wms->wms_default_params); apr_table_setn(params,"BBOX",apr_psprintf(ctx->pool,"%f,%f,%f,%f", map->extent.minx,map->extent.miny,map->extent.maxx,map->extent.maxy)); @@ -49,16 +64,20 @@ apr_table_setn(params,"SRS",map->grid_link->grid->srs); apr_table_overlap(params,wms->getmap_params,APR_OVERLAP_TABLES_SET); - if(map->dimensions && !apr_is_empty_table(map->dimensions)) { - const apr_array_header_t *elts = apr_table_elts(map->dimensions); + + if(map->dimensions && map->dimensions->nelts>0) { int i; - for(i=0; inelts; i++) { - apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); - apr_table_setn(params,entry.key,entry.val); + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(map->dimensions,i,mapcache_requested_dimension*); + /* set both DIM_key=val and key=val KVP params */ + apr_table_setn(params,rdim->dimension->name,rdim->cached_value); + if(strcasecmp(rdim->dimension->name,"TIME") && strcasecmp(rdim->dimension->name,"ELEVATION")) { + char *dim_name = apr_pstrcat(ctx->pool,"DIM_",rdim->dimension->name,NULL); + apr_table_setn(params,dim_name,rdim->cached_value); + } } - } - + /* if the source has no LAYERS parameter defined, then use the tileset name * as the LAYERS to request. When using mirror-mode, the source has no layers * defined, it is added based on the incoming request @@ -68,7 +87,9 @@ } map->encoded_data = mapcache_buffer_create(30000,ctx->pool); - mapcache_http_do_request_with_params(ctx,wms->http,params,map->encoded_data,NULL,NULL); + http = mapcache_http_clone(ctx, wms->http); + http->url = mapcache_http_build_url(ctx,http->url,params); + mapcache_http_do_request(ctx,http,map->encoded_data,NULL,NULL); GC_CHECK_ERROR(ctx); if(!mapcache_imageio_is_valid_format(ctx,map->encoded_data)) { @@ -78,10 +99,11 @@ } } -void _mapcache_source_wms_query(mapcache_context *ctx, mapcache_feature_info *fi) +void _mapcache_source_wms_query(mapcache_context *ctx, mapcache_source *source, mapcache_feature_info *fi) { mapcache_map *map = (mapcache_map*)fi; - mapcache_source_wms *wms = (mapcache_source_wms*)map->tileset->source; + mapcache_http *http; + mapcache_source_wms *wms = (mapcache_source_wms*)source; apr_table_t *params = apr_table_clone(ctx->pool,wms->wms_default_params); apr_table_overlap(params,wms->getmap_params,0); @@ -96,18 +118,24 @@ apr_table_setn(params,"INFO_FORMAT",fi->format); apr_table_overlap(params,wms->getfeatureinfo_params,0); - if(map->dimensions && !apr_is_empty_table(map->dimensions)) { - const apr_array_header_t *elts = apr_table_elts(map->dimensions); + + if(map->dimensions && map->dimensions->nelts>0) { int i; - for(i=0; inelts; i++) { - apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); - apr_table_setn(params,entry.key,entry.val); + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(map->dimensions,i,mapcache_requested_dimension*); + /* set both DIM_key=val and key=val KVP params */ + apr_table_setn(params,rdim->dimension->name,rdim->cached_value); + if(strcasecmp(rdim->dimension->name,"TIME") && strcasecmp(rdim->dimension->name,"ELEVATION")) { + char *dim_name = apr_pstrcat(ctx->pool,"DIM_",rdim->dimension->name,NULL); + apr_table_setn(params,dim_name,rdim->cached_value); + } } - } fi->data = mapcache_buffer_create(30000,ctx->pool); - mapcache_http_do_request_with_params(ctx,wms->http,params,fi->data,NULL,NULL); + http = mapcache_http_clone(ctx, wms->http); + http->url = mapcache_http_build_url(ctx,http->url,params); + mapcache_http_do_request(ctx,http,fi->data,NULL,NULL); GC_CHECK_ERROR(ctx); } @@ -116,7 +144,7 @@ * \private \memberof mapcache_source_wms * \sa mapcache_source::configuration_parse() */ -void _mapcache_source_wms_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source) +void _mapcache_source_wms_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source, mapcache_cfg *config) { ezxml_t cur_node; mapcache_source_wms *src = (mapcache_source_wms*)source; @@ -199,10 +227,10 @@ } mapcache_source_init(ctx, &(source->source)); source->source.type = MAPCACHE_SOURCE_WMS; - source->source.render_map = _mapcache_source_wms_render_map; + source->source._render_map = _mapcache_source_wms_render_map; source->source.configuration_check = _mapcache_source_wms_configuration_check; source->source.configuration_parse_xml = _mapcache_source_wms_configuration_parse_xml; - source->source.query_info = _mapcache_source_wms_query; + source->source._query_info = _mapcache_source_wms_query; source->wms_default_params = apr_table_make(ctx->pool,4);; source->getmap_params = apr_table_make(ctx->pool,4); source->getfeatureinfo_params = apr_table_make(ctx->pool,4); diff -Nru mapcache-1.2.1/lib/strptime.c mapcache-1.6.1/lib/strptime.c --- mapcache-1.2.1/lib/strptime.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/strptime.c 2017-09-29 21:31:13.000000000 +0000 @@ -40,7 +40,6 @@ #include #include "util.h" - static const char *abb_weekdays[] = { "Sun", "Mon", diff -Nru mapcache-1.2.1/lib/tileset.c mapcache-1.6.1/lib/tileset.c --- mapcache-1.2.1/lib/tileset.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/tileset.c 2017-09-29 21:31:13.000000000 +0000 @@ -37,24 +37,22 @@ #include #endif -char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt) -{ +char* mapcache_tileset_tile_resource_key(mapcache_context *ctx, mapcache_tile *tile) { char *lockname = apr_psprintf(ctx->pool, "%d-%d-%d-%s", - mt->z,mt->y,mt->x, - mt->map.tileset->name); + tile->z,tile->y/tile->tileset->metasize_y,tile->x/tile->tileset->metasize_x, + tile->tileset->name); /* if the tileset has multiple grids, add the name of the current grid to the lock key*/ - if(mt->map.tileset->grid_links->nelts > 1) { - lockname = apr_pstrcat(ctx->pool,lockname,mt->map.grid_link->grid->name,NULL); + if(tile->tileset->grid_links->nelts > 1) { + lockname = apr_pstrcat(ctx->pool,lockname,tile->grid_link->grid->name,NULL); } - if(mt->map.dimensions && !apr_is_empty_table(mt->map.dimensions)) { - const apr_array_header_t *elts = apr_table_elts(mt->map.dimensions); + if(tile->dimensions && tile->dimensions->nelts>0) { int i; - for(i=0; inelts; i++) { - apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); - char *dimvalue = apr_pstrdup(ctx->pool,entry.val); + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + char *dimvalue = apr_pstrdup(ctx->pool,rdim->cached_value); char *iter = dimvalue; while(*iter) { if(*iter == '/') *iter='_'; @@ -67,11 +65,16 @@ return lockname; } +char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt) +{ + return mapcache_tileset_tile_resource_key(ctx,&mt->tiles[0]); +} + void mapcache_tileset_configuration_check(mapcache_context *ctx, mapcache_tileset *tileset) { /* check we have all we want */ - if(tileset->cache == NULL) { + if(tileset->_cache == NULL) { /* TODO: we should allow tilesets with no caches */ ctx->set_error(ctx, 400, "tileset \"%s\" has no cache configured.", tileset->name); return; @@ -151,6 +154,30 @@ tileset->watermark = mapcache_imageio_decode(ctx,watermarkdata); } +void mapcache_tileset_tile_validate_z(mapcache_context *ctx, mapcache_tile *tile) { + if(tile->z < tile->grid_link->minz || tile->z >= tile->grid_link->maxz) { + ctx->set_error(ctx,404,"invalid tile z level"); + } +} + +void mapcache_tileset_tile_validate_x(mapcache_context *ctx, mapcache_tile *tile) { + mapcache_extent_i limits; + limits = tile->grid_link->grid_limits[tile->z]; + if(tile->xx>=limits.maxx) { + ctx->set_error(ctx, 404, "tile x=%d not in [%d,%d[", + tile->x,limits.minx,limits.maxx); + } +} + +void mapcache_tileset_tile_validate_y(mapcache_context *ctx, mapcache_tile *tile) { + mapcache_extent_i limits; + limits = tile->grid_link->grid_limits[tile->z]; + if(tile->yy>=limits.maxy) { + ctx->set_error(ctx, 404, "tile y=%d not in [%d,%d[", + tile->y,limits.miny,limits.maxy); + } +} + void mapcache_tileset_tile_validate(mapcache_context *ctx, mapcache_tile *tile) { mapcache_extent_i limits; @@ -176,7 +203,8 @@ mapcache_grid_link *grid_link, mapcache_extent *bbox, int width, int height, int *ntiles, - mapcache_tile ***tiles) + mapcache_tile ***tiles, + mapcache_grid_link **effectively_used_grid_link) { double resolution; int level; @@ -185,25 +213,25 @@ int x,y; int i=0; resolution = mapcache_grid_get_resolution(bbox, width, height); - mapcache_grid_get_closest_level(ctx,grid_link,resolution,&level); - + *effectively_used_grid_link = mapcache_grid_get_closest_wms_level(ctx,grid_link,resolution,&level); + /* we don't want to assemble tiles that have already been reassembled from a lower level */ - if(grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE && level > grid_link->max_cached_zoom) { - level = grid_link->max_cached_zoom; + if((*effectively_used_grid_link)->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE && level > (*effectively_used_grid_link)->max_cached_zoom) { + level = (*effectively_used_grid_link)->max_cached_zoom; } - mapcache_grid_get_xy(ctx,grid_link->grid,bbox->minx,bbox->miny,level,&bl_x,&bl_y); - mapcache_grid_get_xy(ctx,grid_link->grid,bbox->maxx,bbox->maxy,level,&tr_x,&tr_y); - Mx = MAPCACHE_MAX(tr_x,bl_x); - My = MAPCACHE_MAX(tr_y,bl_y); - mx = MAPCACHE_MIN(tr_x,bl_x); - my = MAPCACHE_MIN(tr_y,bl_y); + mapcache_grid_get_xy(ctx,(*effectively_used_grid_link)->grid,bbox->minx,bbox->miny,level,&bl_x,&bl_y); + mapcache_grid_get_xy(ctx,(*effectively_used_grid_link)->grid,bbox->maxx,bbox->maxy,level,&tr_x,&tr_y); + Mx = MAPCACHE_MAX(MAPCACHE_MIN(MAPCACHE_MAX(tr_x,bl_x),(*effectively_used_grid_link)->grid_limits[level].maxx),(*effectively_used_grid_link)->grid_limits[level].minx); + My = MAPCACHE_MAX(MAPCACHE_MIN(MAPCACHE_MAX(tr_y,bl_y),(*effectively_used_grid_link)->grid_limits[level].maxy),(*effectively_used_grid_link)->grid_limits[level].miny); + mx = MAPCACHE_MIN(MAPCACHE_MAX(MAPCACHE_MIN(tr_x,bl_x),(*effectively_used_grid_link)->grid_limits[level].minx),(*effectively_used_grid_link)->grid_limits[level].maxx); + my = MAPCACHE_MIN(MAPCACHE_MAX(MAPCACHE_MIN(tr_y,bl_y),(*effectively_used_grid_link)->grid_limits[level].miny),(*effectively_used_grid_link)->grid_limits[level].maxy); *ntiles = (Mx-mx+1)*(My-my+1); i=0; *tiles = (mapcache_tile**)apr_pcalloc(ctx->pool, *ntiles*sizeof(mapcache_tile*)); for(x=mx; x<=Mx; x++) { for(y=my; y<=My; y++) { - mapcache_tile *tile = mapcache_tileset_tile_create(ctx->pool,tileset, grid_link); + mapcache_tile *tile = mapcache_tileset_tile_create(ctx->pool,tileset, (*effectively_used_grid_link)); tile->x = x; tile->y = y; tile->z = level; @@ -302,6 +330,9 @@ ox = (Mx - tile->x) * tile->grid_link->grid->tile_sx; oy = (tile->y - my) * tile->grid_link->grid->tile_sy; break; + default: + ctx->set_error(ctx,500,"BUG: invalid grid origin"); + return NULL; } if(tile->nodata) continue; @@ -326,7 +357,7 @@ /* copy/scale the srcimage onto the destination image */ tileresolution = toplefttile->grid_link->grid->levels[toplefttile->z]->resolution; - mapcache_grid_get_extent(ctx,toplefttile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx,toplefttile->grid_link->grid, toplefttile->x, toplefttile->y, toplefttile->z, &tilebbox); /*compute the pixel position of top left corner*/ @@ -405,7 +436,7 @@ /* configured metatile size in geographical units */ fullgwidth = res * tileset->metasize_x * grid->tile_sx; fullgheight = res * tileset->metasize_y * grid->tile_sy; - + switch(grid->origin) { case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: mt->map.extent.minx = grid->extent.minx + mt->x * fullgwidth - gbuffer; @@ -448,26 +479,16 @@ */ void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt) { - int i; -#ifdef DEBUG - if(!mt->map.tileset->source || mt->map.tileset->read_only) { - ctx->set_error(ctx,500,"###BUG### tileset_render_metatile called on tileset with no source or that is read-only"); + mapcache_tileset *tileset = mt->map.tileset; + if(!tileset->source || tileset->read_only) { + ctx->set_error(ctx,500,"tileset_render_metatile called on tileset with no source or that is read-only"); return; } -#endif - mt->map.tileset->source->render_map(ctx, &mt->map); + mapcache_source_render_map(ctx, tileset->source, &mt->map); GC_CHECK_ERROR(ctx); mapcache_image_metatile_split(ctx, mt); GC_CHECK_ERROR(ctx); - if(mt->map.tileset->cache->tile_multi_set) { - mt->map.tileset->cache->tile_multi_set(ctx, mt->tiles, mt->ntiles); - } else { - for(i=0; intiles; i++) { - mapcache_tile *tile = &(mt->tiles[i]); - mt->map.tileset->cache->tile_set(ctx, tile); - GC_CHECK_ERROR(ctx); - } - } + mapcache_cache_tile_multi_set(ctx, tileset->_cache, mt->tiles, mt->ntiles); } @@ -487,6 +508,9 @@ tileset->format = NULL; tileset->grid_links = NULL; tileset->config = NULL; + tileset->store_dimension_assemblies = 1; + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_NONE; + tileset->subdimension_read_only = 0; return tileset; } @@ -504,11 +528,14 @@ dst->grid_links = src->grid_links; dst->config = src->config; dst->name = src->name; - dst->cache = src->cache; + dst->_cache = src->_cache; dst->source = src->source; dst->watermark = src->watermark; dst->wgs84bbox = src->wgs84bbox; dst->format = src->format; + dst->store_dimension_assemblies = src->store_dimension_assemblies; + dst->dimension_assembly_type = src->dimension_assembly_type; + dst->subdimension_read_only = src->subdimension_read_only; return dst; } @@ -527,17 +554,15 @@ tile->grid_link = grid_link; if(tileset->dimensions) { int i; - tile->dimensions = apr_table_make(pool,tileset->dimensions->nelts); + tile->dimensions = apr_array_make(pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); for(i=0; idimensions->nelts; i++) { mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - apr_table_set(tile->dimensions,dimension->name,dimension->default_value); - } - } - if(tileset->timedimension) { - if(!tile->dimensions) { - tile->dimensions = apr_table_make(pool,1); + mapcache_requested_dimension *rdim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + rdim->requested_value = dimension->default_value; + rdim->cached_value = NULL; + rdim->dimension = dimension; + APR_ARRAY_PUSH(tile->dimensions,mapcache_requested_dimension*) = rdim; } - apr_table_set(tile->dimensions,tileset->timedimension->key,tileset->timedimension->default_value); } return tile; } @@ -548,12 +573,11 @@ tile->tileset = src->tileset; tile->expires = src->expires; tile->grid_link = src->grid_link; - if(src->dimensions) { - tile->dimensions = apr_table_clone(pool,src->dimensions); - } + tile->dimensions = mapcache_requested_dimensions_clone(pool, src->dimensions); tile->x = src->x; tile->y = src->y; tile->z = src->z; + tile->allow_redirect = src->allow_redirect; return tile; } @@ -563,7 +587,7 @@ map->tileset = src->tileset; map->expires = src->expires; map->grid_link = src->grid_link; - map->dimensions = apr_table_clone(pool,src->dimensions); + map->dimensions = mapcache_requested_dimensions_clone(pool, src->dimensions); map->height = src->height; map->width = src->width; map->extent = src->extent; @@ -580,17 +604,15 @@ map->grid_link = grid_link; if(tileset->dimensions) { int i; - map->dimensions = apr_table_make(pool,tileset->dimensions->nelts); + map->dimensions = apr_array_make(pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); for(i=0; idimensions->nelts; i++) { mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - apr_table_set(map->dimensions,dimension->name,dimension->default_value); - } - } - if(tileset->timedimension) { - if(!map->dimensions) { - map->dimensions = apr_table_make(pool,1); + mapcache_requested_dimension *rdim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + rdim->requested_value = dimension->default_value; + rdim->cached_value = NULL; + rdim->dimension = dimension; + APR_ARRAY_PUSH(map->dimensions,mapcache_requested_dimension*) = rdim; } - apr_table_set(map->dimensions,tileset->timedimension->key,tileset->timedimension->default_value); } return map; } @@ -606,30 +628,35 @@ fi->map.grid_link = grid_link; if(tileset->dimensions) { int i; - fi->map.dimensions = apr_table_make(pool,tileset->dimensions->nelts); + fi->map.dimensions = apr_array_make(pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); for(i=0; idimensions->nelts; i++) { mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - apr_table_set(fi->map.dimensions,dimension->name,dimension->default_value); + mapcache_requested_dimension *rdim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + rdim->requested_value = dimension->default_value; + rdim->cached_value = NULL; + rdim->dimension = dimension; + APR_ARRAY_PUSH(fi->map.dimensions,mapcache_requested_dimension*) = rdim; } } return fi; } void mapcache_tileset_assemble_out_of_zoom_tile(mapcache_context *ctx, mapcache_tile *tile) { - assert(tile->grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE); - - /* we have at most 4 tiles composing the requested tile */ mapcache_extent tile_bbox; double shrink_x, shrink_y, scalefactor; int x[4],y[4]; int i, n=1; - mapcache_grid_get_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z, &tile_bbox); + mapcache_tile *childtile; + assert(tile->grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE); + + /* we have at most 4 tiles composing the requested tile */ + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z, &tile_bbox); /* shrink the extent so we do not fall exactly on a tile boundary, to avoid rounding errors when computing the x,y of the lower level tile(s) we will need */ - + shrink_x = (tile_bbox.maxx - tile_bbox.minx) / (tile->grid_link->grid->tile_sx * 1000); /* 1/1000th of a pixel */ shrink_y = (tile_bbox.maxy - tile_bbox.miny) / (tile->grid_link->grid->tile_sy * 1000); /* 1/1000th of a pixel */ tile_bbox.maxx -= shrink_x; @@ -654,15 +681,15 @@ tile_bbox.minx -= shrink_x; tile_bbox.miny -= shrink_y; - mapcache_tile *childtile = mapcache_tileset_tile_clone(ctx->pool,tile); + childtile = mapcache_tileset_tile_clone(ctx->pool,tile); childtile->z = tile->grid_link->max_cached_zoom; scalefactor = childtile->grid_link->grid->levels[childtile->z]->resolution/tile->grid_link->grid->levels[tile->z]->resolution; tile->nodata = 1; for(i=0;ix = x[i]; - childtile->y = y[i]; mapcache_extent childtile_bbox; double dstminx,dstminy; + childtile->x = x[i]; + childtile->y = y[i]; mapcache_tileset_tile_get(ctx,childtile); GC_CHECK_ERROR(ctx); if(childtile->nodata) { @@ -681,7 +708,7 @@ tile->nodata = 0; } /* now copy/scale the srcimage onto the destination image */ - mapcache_grid_get_extent(ctx,childtile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx,childtile->grid_link->grid, childtile->x, childtile->y, childtile->z, &childtile_bbox); /*compute the pixel position of top left corner*/ @@ -698,13 +725,14 @@ * https://bugs.freedesktop.org/show_bug.cgi?id=46277 */ unsigned int row,col; unsigned char *srcpixptr; + unsigned char *row_ptr; unsigned int dstminxi = - dstminx / scalefactor; unsigned int dstminyi = - dstminy / scalefactor; srcpixptr = &(childtile->raw_image->data[dstminyi * childtile->raw_image->stride + dstminxi * 4]); /* ctx->log(ctx, MAPCACHE_WARN, "factor: %g. pixel: %d,%d (val:%d)",scalefactor,dstminxi,dstminyi,*((unsigned int*)srcpixptr)); */ - unsigned char *row_ptr = tile->raw_image->data; + row_ptr = tile->raw_image->data; for(row=0;rowraw_image->h;row++) { unsigned char *pix_ptr = row_ptr; for(col=0;colraw_image->w;col++) { @@ -721,11 +749,6 @@ childtile->raw_image = NULL; childtile->encoded_data = NULL; } - - - - - } void mapcache_tileset_outofzoom_get(mapcache_context *ctx, mapcache_tile *tile) { @@ -741,6 +764,228 @@ } } +int mapcache_tileset_tile_get_readonly(mapcache_context *ctx, mapcache_tile *tile) { + int ret = mapcache_cache_tile_get(ctx, tile->tileset->_cache, tile); + if(GC_HAS_ERROR(ctx)) + return ret; + + if(ret == MAPCACHE_SUCCESS && tile->tileset->auto_expire && tile->mtime && tile->tileset->source && !tile->tileset->read_only) { + /* the cache is in auto-expire mode, and can return the tile modification date, + * and there is a source configured so we can possibly update it, + * so we check to see if it is stale */ + apr_time_t now = apr_time_now(); + apr_time_t stale = tile->mtime + apr_time_from_sec(tile->tileset->auto_expire); + if(staleget_error(ctx) == 404) { + ctx->clear_errors(ctx); + } + ret = MAPCACHE_CACHE_MISS; + } + } + return ret; +} + +typedef struct { + mapcache_tile *tile; + int cache_status; +} mapcache_subtile; + +static void mapcache_tileset_tile_get_without_subdimensions(mapcache_context *ctx, mapcache_tile *tile, int read_only); + +void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile) { + apr_array_header_t *subtiles; + mapcache_extent extent; + mapcache_subtile st; + mapcache_image *assembled_image = NULL; + mapcache_buffer *assembled_buffer = NULL; + int i,j,k,n_subtiles = 1,assembled_nodata = 1; + /* we can be here in two cases: + * - either we didn't look up the tile directly (need to split dimension into sub-dimension and reassemble dynamically) + * - either the direct lookup failed and we need to render/assemble the tiles from subdimensions + */ + subtiles = apr_array_make(ctx->pool,1,sizeof(mapcache_subtile)); + st.tile = tile; + APR_ARRAY_PUSH(subtiles,mapcache_subtile) = st; + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + if(GC_HAS_ERROR(ctx)) goto cleanup; + + for(i=0;idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + apr_array_header_t *single_subdimension = rdim->dimension->get_entries_for_value(ctx,rdim->dimension,rdim->requested_value, + tile->tileset, &extent, tile->grid_link->grid); + if(GC_HAS_ERROR(ctx)) /* invalid dimension given */ + goto cleanup; +#ifdef DEBUG + { + char *dims = ""; + int i; + for(i=0;inelts;i++) + dims = apr_pstrcat(ctx->pool,dims,APR_ARRAY_IDX(single_subdimension,i,char*)," ",NULL); + ctx->log(ctx,MAPCACHE_DEBUG,"tile (%d,%d,%d) dimension (%s) returned: %s", + tile->z,tile->y,tile->x,rdim->dimension->name,dims); + } +#endif + + if(single_subdimension->nelts == 0) { + /* not an error, but no subdimension was found: we need to return an empty tile */ + tile->nodata = 1; + if(tile->tileset->store_dimension_assemblies) { + tile->raw_image = mapcache_image_create_with_data(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy); + tile->raw_image->has_alpha = MC_ALPHA_YES; + tile->raw_image->is_blank = MC_EMPTY_YES; + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + /* set the key for the dimension so it can be stored with the requested dimension */ + for(j=0;jdimensions->nelts;j++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,j,mapcache_requested_dimension*); + dim->cached_value = dim->requested_value; + } + mapcache_cache_tile_set(ctx, tile->tileset->_cache, tile); + GC_CHECK_ERROR(ctx); + } + return; + } else { + for(j=0;jnelts;k++) { + st.tile = mapcache_tileset_tile_clone(ctx->pool,APR_ARRAY_IDX(subtiles,j,mapcache_subtile).tile); + APR_ARRAY_PUSH(subtiles,mapcache_subtile)=st; + } + } + n_subtiles *= single_subdimension->nelts; + /* foreach of the subtiles, now set the actual subdimension we are going to be using + the "j%nelts" part takes care of looping over and over through the individual subdimensions */ + for(j=0;jdimension->name, + APR_ARRAY_IDX(single_subdimension,j%single_subdimension->nelts,char*)); + + } + } + } + + /* our subtiles array now contains a list of tiles with subdimensions split up, we now need to fetch them from the cache */ + /* note that subtiles[0].tile == tile */ + + for(i=subtiles->nelts-1; i>=0; i--) { + mapcache_tile *subtile = APR_ARRAY_IDX(subtiles,i,mapcache_subtile).tile; + mapcache_tileset_tile_get_without_subdimensions(ctx, subtile, (tile->tileset->subdimension_read_only||!tile->tileset->source)?1:0); /* creates the tile from the source, takes care of metatiling */ + if(GC_HAS_ERROR(ctx)) + goto cleanup; + if(!subtile->nodata) { + assembled_nodata = 0; + if(!assembled_buffer && !assembled_image) { + /* first "usable" subtile */ + assembled_buffer = subtile->encoded_data; + assembled_image = subtile->raw_image; + } else { + /* need to merge current assembled tile over this subtile */ + if(!assembled_image) { + assembled_image = mapcache_imageio_decode(ctx,assembled_buffer); + if(GC_HAS_ERROR(ctx)) + goto cleanup; + assembled_buffer = NULL; /* the image data went stale as we're merging something */ + } + if(!subtile->raw_image) { + subtile->raw_image = mapcache_imageio_decode(ctx,subtile->encoded_data); + if(GC_HAS_ERROR(ctx)) + goto cleanup; + } + mapcache_image_merge(ctx, subtile->raw_image, assembled_image); + assembled_image = subtile->raw_image; + assembled_image->has_alpha = MC_ALPHA_UNKNOWN; /* we've merged two images, we now have no idea if it's transparent or not */ + if(GC_HAS_ERROR(ctx)) + goto cleanup; + } + if((subtile->encoded_data && mapcache_imageio_header_sniff(ctx,subtile->encoded_data) == GC_JPEG)|| + (subtile->raw_image && subtile->raw_image->has_alpha == MC_ALPHA_NO)) { + /* the returned image is fully opaque, we don't need to get/decode/merge any further subtiles */ + if(assembled_image) + assembled_image->has_alpha = MC_ALPHA_NO; + break; + } + } + } + + tile->encoded_data = assembled_buffer; + tile->raw_image = assembled_image; + tile->nodata = assembled_nodata; + + /* TODO: how should the no data case be handled generically? + * uncomment the following block if this nodata state should be returned to + * the requester immediately, without this info being stored to the cache. + * Leaving this uncommented will cause a no-data tile to be (maybe, depending + * on the cache's actual configuration) written to the cache + */ + /* + if(tile->nodata) { + goto cleanup; + } + */ + + if(!tile->nodata && !tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + if(tile->tileset->store_dimension_assemblies) { + int already_stored = 1; /*depending on the type of dimension, we may have no nead to store the resulting tile*/ + + if(n_subtiles != 1) + already_stored = 0; /*if we had to merge multiple subdimensions, then we always have to store the resulting assembly*/ + + /* set the key for the dimension so it can be stored with the requested dimension */ + for(j=0;jdimensions->nelts;j++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,j,mapcache_requested_dimension*); + if(strcmp(dim->cached_value,dim->requested_value)) { + already_stored = 0; /*the subdimension is different than the requested dimension, we need to store the resulting tile*/ + } + dim->cached_value = dim->requested_value; + } + if(!already_stored) { + if(tile->nodata) { + tile->raw_image = mapcache_image_create_with_data(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy); + tile->raw_image->has_alpha = MC_ALPHA_YES; + tile->raw_image->is_blank = MC_EMPTY_YES; + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + mapcache_cache_tile_set(ctx, tile->tileset->_cache, tile); + GC_CHECK_ERROR(ctx); + } + } + +cleanup: + return; +} + +void mapcache_tileset_tile_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile) { + int i,ret; + assert(tile->dimensions); + if(tile->tileset->store_dimension_assemblies) { + for(i=0;idimensions->nelts;i++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + dim->cached_value = dim->requested_value; + } + ret = mapcache_tileset_tile_get_readonly(ctx,tile); + GC_CHECK_ERROR(ctx); + if(ret == MAPCACHE_SUCCESS) { + /* update the tile expiration time */ + if(tile->tileset->auto_expire && tile->mtime) { + apr_time_t now = apr_time_now(); + apr_time_t expire_time = tile->mtime + apr_time_from_sec(tile->tileset->auto_expire); + tile->expires = apr_time_sec(expire_time-now); + } + return; + } + for(i=0;idimensions->nelts;i++) { + /* unset the cached dimension we setup earlier on */ + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + dim->cached_value = NULL; + } + } + return mapcache_tileset_tile_set_get_with_subdimensions(ctx,tile); + +} + /** * \brief return the image data for a given tile * this call uses a global (interprocess+interthread) mutex if the tile was not found @@ -759,16 +1004,12 @@ * - release mutex * */ -void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile) + +static void mapcache_tileset_tile_get_without_subdimensions(mapcache_context *ctx, mapcache_tile *tile, int read_only) { - int isLocked,ret; + int ret; mapcache_metatile *mt=NULL; - if(tile->grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && - tile->z > tile->grid_link->max_cached_zoom) { - mapcache_tileset_outofzoom_get(ctx, tile); - return; - } - ret = tile->tileset->cache->tile_get(ctx, tile); + ret = mapcache_cache_tile_get(ctx, tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(ret == MAPCACHE_SUCCESS && tile->tileset->auto_expire && tile->mtime && tile->tileset->source && !tile->tileset->read_only) { @@ -778,16 +1019,14 @@ apr_time_t now = apr_time_now(); apr_time_t stale = tile->mtime + apr_time_from_sec(tile->tileset->auto_expire); if(staletileset->read_only || !tile->tileset->source) { + if(read_only) { /* there is no source configured for this tile. not an error, let caller now*/ /* ctx->set_error(ctx,404,"tile not in cache, and no source configured for tileset %s", @@ -802,45 +1041,71 @@ ctx->set_error(ctx,404,"tile not in cache, and configured for readonly mode"); return; } + } - /* the tile does not exist, we must take action before re-asking for it */ - /* - * is the tile already being rendered by another thread ? - * the call is protected by the same mutex that sets the lock on the tile, - * so we can assure that: - * - if the lock does not exist, then this thread should do the rendering - * - if the lock exists, we should wait for the other thread to finish - */ - /* aquire a lock on the metatile */ - mt = mapcache_tileset_metatile_get(ctx, tile); - isLocked = mapcache_lock_or_wait_for_resource(ctx, mapcache_tileset_metatile_resource_key(ctx,mt)); + if (ret == MAPCACHE_CACHE_MISS || ret == MAPCACHE_CACHE_RELOAD) { + int isLocked = MAPCACHE_FALSE; + void *lock; + + /* If the tile does not exist or stale, we must take action before re-asking for it */ + if( !read_only && !ctx->config->non_blocking) { + /* + * is the tile already being rendered by another thread ? + * the call is protected by the same mutex that sets the lock on the tile, + * so we can assure that: + * - if the lock does not exist, then this thread should do the rendering + * - if the lock exists, we should wait for the other thread to finish + */ + /* aquire a lock on the metatile */ + mt = mapcache_tileset_metatile_get(ctx, tile); + isLocked = mapcache_lock_or_wait_for_resource(ctx, ctx->config->locker, mapcache_tileset_metatile_resource_key(ctx,mt), &lock); + GC_CHECK_ERROR(ctx); - if(isLocked == MAPCACHE_TRUE) { - /* no other thread is doing the rendering, do it ourselves */ + + if(isLocked == MAPCACHE_TRUE) { + /* no other thread is doing the rendering, do it ourselves */ #ifdef DEBUG - ctx->log(ctx, MAPCACHE_DEBUG, "cache miss: tileset %s - tile %d %d %d", - tile->tileset->name,tile->x, tile->y,tile->z); + ctx->log(ctx, MAPCACHE_DEBUG, "cache miss/reload: tileset %s - tile %d %d %d", + tile->tileset->name,tile->x, tile->y,tile->z); #endif - /* this will query the source to create the tiles, and save them to the cache */ - mapcache_tileset_render_metatile(ctx, mt); + /* this will query the source to create the tiles, and save them to the cache */ + mapcache_tileset_render_metatile(ctx, mt); - mapcache_unlock_resource(ctx, mapcache_tileset_metatile_resource_key(ctx,mt)); + if(GC_HAS_ERROR(ctx)) { + /* temporarily clear error state so we don't mess up with error handling in the locker */ + void *error; + ctx->pop_errors(ctx,&error); + mapcache_unlock_resource(ctx, ctx->config->locker, lock); + ctx->push_errors(ctx,error); + } else { + mapcache_unlock_resource(ctx, ctx->config->locker, lock); + } + } } - GC_CHECK_ERROR(ctx); - /* the previous step has successfully finished, we can now query the cache to return the tile content */ - ret = tile->tileset->cache->tile_get(ctx, tile); - GC_CHECK_ERROR(ctx); + if (ret == MAPCACHE_CACHE_RELOAD && GC_HAS_ERROR(ctx)) + /* If we tried to reload a stale tile but failed, we know we have already + * fetched it from the cache. We can then ignore errors and just use old tile. + */ + ctx->clear_errors(ctx); - if(ret != MAPCACHE_SUCCESS) { - if(isLocked == MAPCACHE_FALSE) { - ctx->set_error(ctx, 500, "tileset %s: unknown error (another thread/process failed to create the tile I was waiting for)", - tile->tileset->name); - } else { - /* shouldn't really happen, as the error ought to have been caught beforehand */ - ctx->set_error(ctx, 500, "tileset %s: failed to re-get tile %d %d %d from cache after set", tile->tileset->name,tile->x,tile->y,tile->z); + else { + /* Else, check for errors and try to fetch the tile from the cache. + */ + GC_CHECK_ERROR(ctx); + ret = mapcache_cache_tile_get(ctx, tile->tileset->_cache, tile); + GC_CHECK_ERROR(ctx); + + if(ret != MAPCACHE_SUCCESS) { + if(isLocked == MAPCACHE_FALSE) { + ctx->set_error(ctx, 500, "tileset %s: unknown error (another thread/process failed to create the tile I was waiting for)", + tile->tileset->name); + } else { + /* shouldn't really happen, as the error ought to have been caught beforehand */ + ctx->set_error(ctx, 500, "tileset %s: failed to re-get tile %d %d %d from cache after set", tile->tileset->name,tile->x,tile->y,tile->z); + } } } } @@ -852,11 +1117,48 @@ } } +void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile) { + if(tile->grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && + tile->z > tile->grid_link->max_cached_zoom) { + mapcache_tileset_outofzoom_get(ctx, tile); + return; + } + if(tile->dimensions) { + if(tile->tileset->dimension_assembly_type != MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + return mapcache_tileset_tile_get_with_subdimensions(ctx,tile); + } else { + int i; + mapcache_requested_dimension *rdim; + mapcache_extent extent; + + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + for(i=0; idimensions->nelts; i++) { + apr_array_header_t *rdim_vals; + rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim_vals = rdim->dimension->get_entries_for_value(ctx,rdim->dimension,rdim->requested_value, tile->tileset, NULL, tile->grid_link->grid); + GC_CHECK_ERROR(ctx); + if(rdim_vals->nelts > 1) { + ctx->set_error(ctx,500,"dimension (%s) for tileset (%s) returned invalid number (%d) of subdimensions (1 expected)", + rdim->dimension->name, tile->tileset->name, rdim_vals->nelts); + return; + } + if(rdim_vals->nelts == 0) { + ctx->set_error(ctx,404,"dimension (%s) for tileset (%s) returned no subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return; + } + rdim->cached_value = APR_ARRAY_IDX(rdim_vals,0,char*); + } + } + } + return mapcache_tileset_tile_get_without_subdimensions(ctx,tile, (tile->tileset->read_only||!tile->tileset->source)?1:0); + +} + void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile) { int i; /*delete the tile itself*/ - tile->tileset->cache->tile_delete(ctx,tile); + mapcache_cache_tile_delete(ctx,tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(whole_metatile) { @@ -865,7 +1167,7 @@ mapcache_tile *subtile = &mt->tiles[i]; /* skip deleting the actual tile */ if(subtile->x == tile->x && subtile->y == tile->y) continue; - subtile->tileset->cache->tile_delete(ctx,subtile); + mapcache_cache_tile_delete(ctx,subtile->tileset->_cache,subtile); /* silently pass failure if the tile was not found */ if(ctx->get_error(ctx) == 404) { ctx->clear_errors(ctx); diff -Nru mapcache-1.2.1/lib/util.c mapcache-1.6.1/lib/util.c --- mapcache-1.2.1/lib/util.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/lib/util.c 2017-09-29 21:31:13.000000000 +0000 @@ -33,6 +33,7 @@ #include #include #include +#include #ifndef _WIN32 #include @@ -41,8 +42,63 @@ #ifndef M_PI #define M_PI 3.14159265358979323846264338327 #endif + +#ifdef _WIN32 +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long int uint64_t; +#endif + const double mapcache_meters_per_unit[MAPCACHE_UNITS_COUNT] = {1.0,6378137.0 * 2.0 * M_PI / 360,0.3048}; + +static char encoding_table[] = {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', + 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', + 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', + 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', + 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', + 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', + 'w', 'x', 'y', 'z', '0', '1', '2', '3', + '4', '5', '6', '7', '8', '9', '+', '/'}; +static int mod_table[] = {0, 2, 1}; + + +char *base64_encode(apr_pool_t *pool, const unsigned char *data, size_t input_length) { + int i,j; + char *encoded_data; + size_t output_length = 4 * ((input_length + 2) / 3) + 1; + + encoded_data = (char*)apr_pcalloc(pool,output_length*sizeof(char)); + if (encoded_data == NULL) return NULL; + + for (i = 0, j = 0; i < input_length;) { + + uint32_t octet_a; + uint32_t octet_b; + uint32_t octet_c; + uint32_t triple; + octet_a = i < input_length ? (unsigned char)data[i++] : 0; + octet_b = i < input_length ? (unsigned char)data[i++] : 0; + octet_c = i < input_length ? (unsigned char)data[i++] : 0; + + triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + + encoded_data[j++] = encoding_table[(triple >> 3 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 2 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 1 * 6) & 0x3F]; + encoded_data[j++] = encoding_table[(triple >> 0 * 6) & 0x3F]; + } + + for (i = 0; i < mod_table[input_length % 3]; i++) + encoded_data[output_length - 2 - i] = '='; + + encoded_data[output_length-1]=0; + + return encoded_data; +} + + int mapcache_util_extract_int_list(mapcache_context *ctx, const char* cargs, const char *sdelim, int **numbers, int *numbers_count) { @@ -129,6 +185,53 @@ return pstr; } +char* mapcache_util_str_xml_escape(apr_pool_t *pool, const char *str, + mapcache_util_xml_section_type xml_section_type) +{ + int outpos = 0; + char* outstr = apr_pcalloc(pool, 6 * strlen(str) + 1); + for( ; *str != '\0'; str ++ ) { + if( xml_section_type == MAPCACHE_UTIL_XML_SECTION_COMMENT ) { + if( *str == '-' ) { + memcpy(outstr + outpos, "-", 5); + outpos += 5; + } + else { + outstr[outpos] = *str; + outpos ++; + } + } + else { + if( *str == '&' ) { + memcpy(outstr + outpos, "&", 5); + outpos += 5; + } + else if( *str == '<' ) { + memcpy(outstr + outpos, "<", 4); + outpos += 4; + } + else if( *str == '>' ) { + memcpy(outstr + outpos, ">", 4); + outpos += 4; + } + else if( *str == '"' ) { + memcpy(outstr + outpos, """, 6); + outpos += 6; + } + else if( *str == '\'' ) { + /* See https://github.com/mapserver/mapserver/issues/1040 */ + memcpy(outstr + outpos, "'", 5); + outpos += 5; + } + else { + outstr[outpos] = *str; + outpos ++; + } + } + } + return outstr; +} + #if APR_MAJOR_VERSION < 1 || (APR_MAJOR_VERSION < 2 && APR_MINOR_VERSION < 3) APR_DECLARE(apr_table_t *) apr_table_clone(apr_pool_t *p, const apr_table_t *t) { @@ -172,18 +275,18 @@ void _mapcache_context_set_error_default(mapcache_context *ctx, int code, char *msg, ...) { - char *fmt; + char *new_msg; va_list args; va_start(args,msg); + new_msg = apr_pvsprintf(ctx->pool,msg,args); + va_end(args); if(ctx->_errmsg) { - fmt=apr_psprintf(ctx->pool,"%s\n%s",ctx->_errmsg,msg); + ctx->_errmsg = apr_pstrcat(ctx->pool, ctx->_errmsg, "\n", new_msg, NULL); } else { - fmt=msg; + ctx->_errmsg = new_msg; ctx->_errcode = code; } - ctx->_errmsg = apr_pvsprintf(ctx->pool,fmt,args); - va_end(args); } void _mapcache_context_clear_error_default(mapcache_context *ctx) @@ -196,6 +299,47 @@ } +struct _error_log { + int _errcode; + char *_errmsg; + apr_table_t *exceptions; +}; + +void _mapcache_context_pop_errors(mapcache_context *ctx, void **error) +{ + struct _error_log *e = (struct _error_log*)apr_pcalloc(ctx->pool, sizeof(struct _error_log)); + e->_errcode = ctx->_errcode; + e->_errmsg = ctx->_errmsg; + e->exceptions = ctx->exceptions; + ctx->_errcode = 0; + ctx->_errmsg = NULL; + ctx->exceptions = NULL; + *error = e; +} + + +void _mapcache_context_push_errors(mapcache_context *ctx, void *error) +{ + struct _error_log *e = (struct _error_log*)error; + if(e->_errcode) + ctx->_errcode = e->_errcode; + if(e->_errmsg) { + if(ctx->_errmsg) { + ctx->_errmsg = apr_psprintf(ctx->pool,"%s\n%s",e->_errmsg,ctx->_errmsg); + } else { + ctx->_errmsg = e->_errmsg; + } + } + if(e->exceptions) { + if(ctx->exceptions) { + apr_table_overlap(ctx->exceptions, e->exceptions, APR_OVERLAP_TABLES_SET); + } else { + ctx->exceptions = e->exceptions; + } + } +} + + void mapcache_context_init(mapcache_context *ctx) { ctx->_errcode = 0; @@ -205,6 +349,9 @@ ctx->set_error = _mapcache_context_set_error_default; ctx->set_exception = _mapcache_context_set_exception_default; ctx->clear_errors = _mapcache_context_clear_error_default; + ctx->pop_errors = _mapcache_context_pop_errors; + ctx->push_errors = _mapcache_context_push_errors; + ctx->headers_in = NULL; } void mapcache_context_copy(mapcache_context *src, mapcache_context *dst) @@ -224,29 +371,31 @@ dst->set_exception = src->set_exception; dst->service = src->service; dst->exceptions = src->exceptions; - dst->threadlock = src->threadlock; - dst->process_pool = src->process_pool; + dst->supports_redirects = src->supports_redirects; + dst->pop_errors = src->pop_errors; + dst->push_errors = src->push_errors; + dst->connection_pool = src->connection_pool; + dst->headers_in = src->headers_in; } char* mapcache_util_get_tile_dimkey(mapcache_context *ctx, mapcache_tile *tile, char* sanitized_chars, char *sanitize_to) { char *key = apr_pstrdup(ctx->pool,""); if(tile->dimensions) { - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; if(i>1) { while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); if(i) { - key = apr_pstrcat(ctx->pool,key,entry->val,(sanitized_chars?sanitize_to:"#"),NULL); + key = apr_pstrcat(ctx->pool,key,entry->cached_value,(sanitized_chars?sanitize_to:"#"),NULL); } else { - key = apr_pstrcat(ctx->pool,key,entry->val,NULL); + key = apr_pstrcat(ctx->pool,key,entry->cached_value,NULL); } } return key; } else if(i) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,0,apr_table_entry_t)); - key = apr_pstrdup(ctx->pool,entry->val); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,0,mapcache_requested_dimension*); + key = apr_pstrdup(ctx->pool,entry->cached_value); } if(sanitized_chars) key = mapcache_util_str_sanitize(ctx->pool,key,sanitized_chars,*sanitize_to); @@ -254,19 +403,109 @@ return key; } +void mapcache_util_quadkey_decode(mapcache_context *ctx, const char *quadkey, int *x, int *y, int *z) { + int i; + if(!quadkey || !*quadkey) { + *z = *x = *y = 0; + return; + } + *z = strlen(quadkey); + *x = *y = 0; + for (i = *z; i; i--) { + int mask = 1 << (i - 1); + switch (quadkey[*z - i]) { + case '0': + break; + case '1': + *x |= mask; + break; + case '2': + *y |= mask; + break; + case '3': + *x |= mask; + *y |= mask; + break; + default: + ctx->set_error(ctx, 400, "Invalid Quadkey sequence"); + return; + } + } +} + +char* mapcache_util_quadkey_encode(mapcache_context *ctx, int x, int y, int z) { + int i; + char *key = apr_pcalloc(ctx->pool, z+1); + memset(key,'0',z); + for (i = z; i > 0; i--) { + int mask = 1 << (i - 1); + if ((x & mask) != 0) + { + key[z - i]++; + } + if ((y & mask) != 0) + { + key[z - i] += 2; + } + } + return key; +} + char* mapcache_util_get_tile_key(mapcache_context *ctx, mapcache_tile *tile, char *template, char* sanitized_chars, char *sanitize_to) { char *path; if(template) { - path = mapcache_util_str_replace(ctx->pool, template, "{x}", - apr_psprintf(ctx->pool, "%d", tile->x)); - path = mapcache_util_str_replace(ctx->pool, path, "{y}", - apr_psprintf(ctx->pool, "%d", tile->y)); - path = mapcache_util_str_replace(ctx->pool, path, "{z}", - apr_psprintf(ctx->pool, "%d", tile->z)); - if(strstr(path,"{dim}")) { - path = mapcache_util_str_replace(ctx->pool, path, "{dim}", mapcache_util_get_tile_dimkey(ctx,tile,sanitized_chars,sanitize_to)); + path = apr_pstrdup(ctx->pool, template); + + if(strstr(path,"{x}")) + path = mapcache_util_str_replace(ctx->pool,path, "{x}", + apr_psprintf(ctx->pool,"%d",tile->x)); + else if(strstr(path,"{inv_x}")) + path = mapcache_util_str_replace(ctx->pool,path, "{inv_x}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)); + if(strstr(path,"{y}")) + path = mapcache_util_str_replace(ctx->pool,path, "{y}", + apr_psprintf(ctx->pool,"%d",tile->y)); + else if(strstr(path,"{inv_y}")) + path = mapcache_util_str_replace(ctx->pool,path, "{inv_y}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)); + if(strstr(path,"{z}")) + path = mapcache_util_str_replace(ctx->pool,path, "{z}", + apr_psprintf(ctx->pool,"%d",tile->z)); + else if(strstr(path,"{inv_z}")) + path = mapcache_util_str_replace(ctx->pool,path, "{inv_z}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->nlevels - tile->z - 1)); + if(strstr(path,"{quadkey}")) { + char *quadkey = mapcache_util_quadkey_encode(ctx, tile->x, tile->y, tile->z); + path = mapcache_util_str_replace(ctx->pool,path, "{quadkey}", quadkey); + } + + if(tile->dimensions) { + if(strstr(path,"{dim:")) { + char *dimstring=""; + int i = tile->dimensions->nelts; + while(i--) { + char *single_dim; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + + /* compute value for eventual {dim} replacement */ + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",entry->cached_value,NULL); + + /* check for {dim:name} replacement */ + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(path,single_dim)) { + path = mapcache_util_str_replace(ctx->pool,path, single_dim, entry->cached_value); + } + } + } + if(strstr(path,"{dim}")) { + path = mapcache_util_str_replace(ctx->pool,path, "{dim}", + mapcache_util_get_tile_dimkey(ctx,tile,sanitized_chars,sanitize_to)); + } } if(strstr(path,"{tileset}")) path = mapcache_util_str_replace(ctx->pool, path, "{tileset}", tile->tileset->name); @@ -297,9 +536,41 @@ return path; } - -/* vim: ts=2 sts=2 et sw=2 -*/ +void mapcache_make_parent_dirs(mapcache_context *ctx, char *filename) { + char *hackptr1,*hackptr2=NULL; + apr_status_t ret; + char errmsg[120]; + + /* find the location of the last '/' in the string */ + hackptr1 = filename; + while(*hackptr1) { + if(*hackptr1 == '/') + hackptr2 = hackptr1; + hackptr1++; + } + + if(hackptr2) { + /* terminate string on last '/' */ + *hackptr2 = '\0'; + } + + ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool); + + if(hackptr2) { + *hackptr2 = '/'; + } + + + if(APR_SUCCESS != ret) { + /* + * apr_dir_make_recursive sometimes sends back this error, although it should not. + * ignore this one + */ + if(!APR_STATUS_IS_EEXIST(ret)) { + ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); + } + } +} #if defined(_WIN32) && !defined(__CYGWIN__) @@ -353,3 +624,8 @@ #endif + +/* vim: ts=2 sts=2 et sw=2 +*/ + + diff -Nru mapcache-1.2.1/mapcache.xml mapcache-1.6.1/mapcache.xml --- mapcache-1.2.1/mapcache.xml 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/mapcache.xml 2017-09-29 21:31:13.000000000 +0000 @@ -7,6 +7,10 @@ /tmp + + /tmp/{tileset}-{z}-{grid}.db + + @@ -23,9 +27,9 @@ vmap0 - disk + sqlite WGS84 - g + GoogleMapsCompatible PNG 5 5 10 @@ -49,6 +53,9 @@ report - /tmp + + /tmp + 300 + diff -Nru mapcache-1.2.1/mapcache.xml.sample mapcache-1.6.1/mapcache.xml.sample --- mapcache-1.2.1/mapcache.xml.sample 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/mapcache.xml.sample 2017-09-29 21:31:13.000000000 +0000 @@ -73,6 +73,16 @@ + + value + + + create table if not exists tiles(tileset text, grid text, x integer, y integer, z integer, data blob, dim text, ctime datetime, primary key(tileset,grid,x,y,z,dim)) + select 1 from tiles where x=:x and y=:y and z=:z and dim=:dim and tileset=:tileset and grid=:grid + select data,strftime("%s",ctime) from tiles where tileset=:tileset and grid=:grid and x=:x and y=:y and z=:z and dim=:dim + insert or replace into tiles(tileset,grid,x,y,z,data,dim,ctime) values (:tileset,:grid,:x,:y,:z,:data,:dim,datetime('now')) + delete from tiles where x=:x and y=:y and z=:z and dim=:dim and tileset=:tileset and grid=:grid + {tileset}-{grid}-{dim}-{z}-{y}-{x}.{ext} + + + https://myserver/webdav/{tileset}/{grid}/{z}/{x}/{y}.{ext} + + my-virtualhost-alias.domain.com + + + + foo + + + + + foo + + + + + foo + + + + + foo + + + + + https://foo.s3.amazonaws.com/tiles/{tileset}/{grid}/{z}/{x}/{y}/{ext} + + foo.s3.amazonaws.com + + foo + foo/sdsvd + eu-west-1 + + + REDUCED_REDUNDANCY + public-read + + + + + https://foo.blob.core.windows.net/tiles/{tileset}/{grid}/{z}/{x}/{y}/{ext} + + foo.blob.core.windows.net + + foo + foobarcccccccccccccccccccccyA== + tiles + + + + https://storage.googleapis.com/mytiles-mapcache/{tileset}/{grid}/{z}/{x}/{y}.{ext} + GOOGPGDWFDG345SDFGSD + sdfgsdSDFwedfwefr2345324dfsGdsfgs + + + public-read + + + + + + + + + + + + + + /dev/shm/google-mapcache.header + 10 + 3 + + + + + + + + GOOGPGDWFDG345SDFGSD + sdfgsdSDFwedfwefr2345324dfsGdsfgs + 10 + 3 + + + true best @@ -876,12 +986,33 @@ report + + - /tmp + /tmp + 0.01 + + + /tmp + + + + + localhost + 11211 + + + memcache-host + 11212 + + 0.3 + + + true diff -Nru mapcache-1.2.1/MIGRATION_GUIDE.txt mapcache-1.6.1/MIGRATION_GUIDE.txt --- mapcache-1.2.1/MIGRATION_GUIDE.txt 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/MIGRATION_GUIDE.txt 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,16 @@ +Migrating from Mapcache 1.4 to 1.6 +=================================== + +* The tileset child has been removed. Time dimensions are now added with + +* val1,val2,val3 should be replaced by + val1val2val3 + +* ... should be replaced by + true.... + +* ^abc$ should be replaced by + ^abc$ + +* ... should be replaced by + stack.... diff -Nru mapcache-1.2.1/nginx/ngx_http_mapcache_module.c mapcache-1.6.1/nginx/ngx_http_mapcache_module.c --- mapcache-1.2.1/nginx/ngx_http_mapcache_module.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/nginx/ngx_http_mapcache_module.c 2017-09-29 21:31:13.000000000 +0000 @@ -62,8 +62,7 @@ apr_pool_create(&process_pool,NULL); mapcache_context *ctx = apr_pcalloc(process_pool, sizeof(mapcache_ngx_context)); ctx->pool = process_pool; - ctx->process_pool = process_pool; - ctx->threadlock = NULL; + ctx->connection_pool = NULL; mapcache_context_init(ctx); ctx->log = ngx_mapcache_context_log; ctx->clone = ngx_mapcache_context_clone; @@ -211,7 +210,6 @@ mapcache_ngx_context *ngctx = ngx_http_get_module_loc_conf(r, ngx_http_mapcache_module); mapcache_context *ctx = (mapcache_context*)ngctx; apr_pool_create(&(ctx->pool),process_pool); - ctx->process_pool = process_pool; ngctx->r = r; mapcache_request *request = NULL; mapcache_http_response *http_response; @@ -298,6 +296,11 @@ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,ctx->get_error_message(ctx)); return NGX_CONF_ERROR; } + if(mapcache_config_services_enabled(ctx, ctx->config) <= 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "no mapcache s configured/enabled, no point in continuing."); + return NGX_CONF_ERROR; + } + mapcache_connection_pool_create(&ctx->connection_pool,ctx->pool); ctx->config->non_blocking = 1; ngx_http_core_loc_conf_t *clcf; diff -Nru mapcache-1.2.1/release.sh mapcache-1.6.1/release.sh --- mapcache-1.2.1/release.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/release.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,54 @@ +#!/bin/bash + +ms_version=$1 + +echo "release steps for mapcache version $ms_version (mapcache-$ms_version.tar.gz)" +echo "" + +ms_version_suffix=`echo $ms_version | cut -s -d- -f2` +if [ -z $ms_version_suffix ]; then + ms_version_num=$ms_version +else + ms_version_num=`echo $ms_version | cut -s -d- -f1` +fi + +ms_version_major=`echo $ms_version_num | cut -d. -f1` +ms_version_minor=`echo $ms_version_num | cut -d. -f2` +ms_version_revision=`echo $ms_version_num | cut -d. -f3` + + +tagname=rel-$ms_version_major-$ms_version_minor-$ms_version_revision +if [ ! -z $ms_version_suffix ]; then + tagname=$tagname-$ms_version_suffix +fi + +echo "#" +echo "# make sure:" +echo "# - you are on branch-$ms_version_major-$ms_version_minor" +echo "# - you have edited HISTORY.TXT with changes related to this release" +echo "#" +echo "" + +echo "sed -i '/set (MAPCACHE_VERSION_MAJOR/c\set (MAPCACHE_VERSION_MAJOR $ms_version_major)' CMakeLists.txt" +echo "sed -i '/set (MAPCACHE_VERSION_MINOR/c\set (MAPCACHE_VERSION_MINOR $ms_version_minor)' CMakeLists.txt" +echo "sed -i '/set (MAPCACHE_VERSION_REVISION/c\set (MAPCACHE_VERSION_REVISION $ms_version_revision)' CMakeLists.txt" +if [ ! -z $ms_version_suffix ]; then + echo "sed -i '/set (MAPCACHE_VERSION_SUFFIX/c\set (MAPCACHE_VERSION_SUFFIX \"-$ms_version_suffix\")' CMakeLists.txt" +else + echo "sed -i '/set (MAPCACHE_VERSION_SUFFIX/c\set (MAPCACHE_VERSION_SUFFIX \"\")' CMakeLists.txt" +fi + +echo "git add CMakeLists.txt" +echo "git commit -m \"update for $ms_version release\"" +echo "git tag -a $tagname -m \"Create $ms_version tag\"" +echo "git push origin branch-$ms_version_major-$ms_version_minor --tags" +echo "git archive --format=tar.gz --prefix=mapcache-$ms_version/ $tagname >/tmp/mapcache-$ms_version.tar.gz" +echo "scp /tmp/mapcache-$ms_version.tar.gz download.osgeo.org:/osgeo/download/mapserver" + +echo "" +echo "#" +echo "#optionally update doc site, these commands need tweaking before being ran" +echo "#" + +echo "/path/to/docs/scripts/changelog.sh rel-previous-tag..$tagname >> /path/to/docs/en/development/changelog/changelog-$ms_version_major-$ms_version_minor.txt" + diff -Nru mapcache-1.2.1/scripts/vagrant/mapcache.sh mapcache-1.6.1/scripts/vagrant/mapcache.sh --- mapcache-1.2.1/scripts/vagrant/mapcache.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/scripts/vagrant/mapcache.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,13 @@ +#!/bin/sh + +NUMTHREADS=2 # we have 2 cpus configured +export NUMTHREADS + +cd /vagrant + +mkdir build_vagrant +cd build_vagrant +cmake -DWITH_MEMCACHE=1 .. + +make -j $NUMTHREADS +make install diff -Nru mapcache-1.2.1/scripts/vagrant/packages.sh mapcache-1.6.1/scripts/vagrant/packages.sh --- mapcache-1.2.1/scripts/vagrant/packages.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/scripts/vagrant/packages.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,17 @@ +#!/bin/sh + +sed -i 's#deb http://us.archive.ubuntu.com/ubuntu/#deb mirror://mirrors.ubuntu.com/mirrors.txt#' /etc/apt/sources.list + +export DEBIAN_FRONTEND=noninteractive + +apt-get update +apt-get install -y python-software-properties +add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable +apt-get update +apt-get -y upgrade + +# install packages we need +apt-get install -q -y build-essential pkg-config cmake libgeos-dev rake vim \ + bison flex libgdal1-dev libproj-dev libpng12-dev libjpeg-dev libfcgi-dev \ + libcurl4-gnutls-dev apache2-prefork-dev libtiff4-dev libpixman-1-dev \ + libsqlite3-dev libmemcached-dev diff -Nru mapcache-1.2.1/scripts/vagrant/virtualbox-fix.sh mapcache-1.6.1/scripts/vagrant/virtualbox-fix.sh --- mapcache-1.2.1/scripts/vagrant/virtualbox-fix.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/scripts/vagrant/virtualbox-fix.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ ! -e "/usr/lib/VBoxGuestAdditions" ]; then + ln -s /opt/VBoxGuestAdditions-4.3.10/lib/VBoxGuestAdditions /usr/lib/VBoxGuestAdditions +fi Binary files /tmp/tmp5rT7aD/IIM7OEOJT1/mapcache-1.2.1/tests/data/world.tif and /tmp/tmp5rT7aD/WRkXrXsxVo/mapcache-1.6.1/tests/data/world.tif differ diff -Nru mapcache-1.2.1/tests/expected/wms_capabilities.xml mapcache-1.6.1/tests/expected/wms_capabilities.xml --- mapcache-1.2.1/tests/expected/wms_capabilities.xml 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/expected/wms_capabilities.xml 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,75 @@ + + +]> + + + OGC:WMS + no title set, add some in metadata + + + + + + application/vnd.ogc.wms_xml + + + + + + + + + + image/png + image/jpeg + + + + + + + + + + text/plain + application/vnd.ogc.gml + + + + + + + + + + + text/plain + + + + EPSG:3857 + + 156543.03392804099712520838 78271.51696402048401068896 39135.75848201022745342925 19567.87924100512100267224 9783.93962050256050133612 4891.96981025128025066806 2445.98490512564012533403 1222.99245256282006266702 611.49622628141003133351 305.74811314070478829308 152.87405657035250783338 76.43702828517623970583 38.21851414258812695834 19.10925707129405992646 9.55462853564703173959 4.77731426782351586979 2.38865713391175793490 1.19432856695587897633 + 256 + 256 + image/jpeg + global + + + + + no title set, add some in metadata + EPSG:3857 + EPSG:4326 + EPSG:900913 + + global + global + + EPSG:3857 + EPSG:900913 + + + + diff -Nru mapcache-1.2.1/tests/expected/wmts_capabilities.xml mapcache-1.6.1/tests/expected/wmts_capabilities.xml --- mapcache-1.2.1/tests/expected/wmts_capabilities.xml 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/expected/wmts_capabilities.xml 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,247 @@ + + + + OGC WMTS + 1.0.0 + + + + + + + + + + + + + + KVP + + + + + + + + + + + + + KVP + + + + + + + + + + + + + KVP + + + + + + + + + + global + global + + image/jpeg + + GoogleMapsCompatible + + + + + GoogleMapsCompatible + + -20037508.342789 -20037508.342789 + 20037508.342789 20037508.342789 + + urn:ogc:def:crs:EPSG:6.3:3857 + urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible + + 0 + 559082264.02871787548065185547 + -20037508.342789 20037508.342789 + 256 + 256 + 1 + 1 + + + 1 + 279541132.01435887813568115234 + -20037508.342789 20037508.342789 + 256 + 256 + 2 + 2 + + + 2 + 139770566.00717940926551818848 + -20037508.342789 20037508.342789 + 256 + 256 + 4 + 4 + + + 3 + 69885283.00358971953392028809 + -20037508.342789 20037508.342789 + 256 + 256 + 8 + 8 + + + 4 + 34942641.50179485976696014404 + -20037508.342789 20037508.342789 + 256 + 256 + 16 + 16 + + + 5 + 17471320.75089742988348007202 + -20037508.342789 20037508.342789 + 256 + 256 + 32 + 32 + + + 6 + 8735660.37544871494174003601 + -20037508.342789 20037508.342789 + 256 + 256 + 64 + 64 + + + 7 + 4367830.18772435747087001801 + -20037508.342789 20037508.342789 + 256 + 256 + 128 + 128 + + + 8 + 2183915.09386217873543500900 + -20037508.342789 20037508.342789 + 256 + 256 + 256 + 256 + + + 9 + 1091957.54693108866922557354 + -20037508.342789 20037508.342789 + 256 + 256 + 512 + 512 + + + 10 + 545978.77346554468385875225 + -20037508.342789 20037508.342789 + 256 + 256 + 1024 + 1024 + + + 11 + 272989.38673277228372171521 + -20037508.342789 20037508.342789 + 256 + 256 + 2048 + 2048 + + + 12 + 136494.69336638617096468806 + -20037508.342789 20037508.342789 + 256 + 256 + 4096 + 4096 + + + 13 + 68247.34668319307093042880 + -20037508.342789 20037508.342789 + 256 + 256 + 8192 + 8192 + + + 14 + 34123.67334159654274117202 + -20037508.342789 20037508.342789 + 256 + 256 + 16384 + 16384 + + + 15 + 17061.83667079827137058601 + -20037508.342789 20037508.342789 + 256 + 256 + 32768 + 32768 + + + 16 + 8530.91833539913568529300 + -20037508.342789 20037508.342789 + 256 + 256 + 65536 + 65536 + + + 17 + 4265.45916769956784264650 + -20037508.342789 20037508.342789 + 256 + 256 + 131072 + 131072 + + + 18 + 2132.72958384978392132325 + -20037508.342789 20037508.342789 + 256 + 256 + 262144 + 262144 + + + + diff -Nru mapcache-1.2.1/tests/run_tests.sh mapcache-1.6.1/tests/run_tests.sh --- mapcache-1.2.1/tests/run_tests.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/run_tests.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,49 @@ +#!/bin/sh + +# Project: MapCache +# Purpose: MapCache tests +# Author: Even Rouault +# +#***************************************************************************** +# Copyright (c) 2017 Regents of the University of Minnesota. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +#****************************************************************************/ + +set -e + +MAPCACHE_CONF=/tmp/mc/mapcache.xml + +sudo rm -rf /tmp/mc/global +mapcache_seed -c /tmp/mc/mapcache.xml -t global --force -z 0,1 +gdalinfo -checksum /tmp/mc/global/GoogleMapsCompatible/00/000/000/000/000/000/000.jpg | grep Checksum=20574 >/dev/null || (echo "Did not get expected checksum"; gdalinfo -checksum /tmp/mc/global/GoogleMapsCompatible/00/000/000/000/000/000/000.jpg; /bin/false) +sudo rm -rf /tmp/mc/global + +curl -s "http://localhost/mapcache/?SERVICE=WMS&REQUEST=GetCapabilities" | xmllint --format - > /tmp/wms_capabilities.xml +diff -u /tmp/wms_capabilities.xml expected + +curl -s "http://localhost/mapcache/wmts?SERVICE=WMTS&REQUEST=GetCapabilities" | xmllint --format - > /tmp/wmts_capabilities.xml +diff -u /tmp/wmts_capabilities.xml expected + +curl -s "http://localhost/mapcache/wmts/1.0.0/global/default/GoogleMapsCompatible/0/0/0.jpg" > /tmp/0.jpg +gdalinfo -checksum /tmp/0.jpg | grep Checksum=20574 >/dev/null || (echo "Did not get expected checksum"; gdalinfo -checksum /tmp/0.jpg; /bin/false) + +curl -s "http://localhost/mapcache/wmts/1.0.0/global/default/GoogleMapsCompatible/0/0/0.jpg" > /tmp/0_bis.jpg +diff /tmp/0.jpg /tmp/0_bis.jpg + diff -Nru mapcache-1.2.1/tests/travis_setup.sh mapcache-1.6.1/tests/travis_setup.sh --- mapcache-1.2.1/tests/travis_setup.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/travis_setup.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,65 @@ +#!/bin/sh + +# Project: MapCache +# Purpose: MapCache tests +# Author: Even Rouault +# +#***************************************************************************** +# Copyright (c) 2017 Regents of the University of Minnesota. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +#****************************************************************************/ + +set -e + +mkdir /tmp/mc +sudo chmod -R a+rw /tmp/mc + +MAPCACHE_CONF=/tmp/mc/mapcache.xml +echo '' >> $MAPCACHE_CONF +echo '' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' /tmp/mc/world.tif' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' /tmp/mc' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' disk' >> $MAPCACHE_CONF +echo ' global-tif' >> $MAPCACHE_CONF +echo ' GoogleMapsCompatible' >> $MAPCACHE_CONF +echo ' JPEG' >> $MAPCACHE_CONF +echo ' 1 1' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' debug' >> $MAPCACHE_CONF +echo '' >> $MAPCACHE_CONF + +cp data/world.tif /tmp/mc + +sudo su -c "echo 'LoadModule mapcache_module /usr/lib/apache2/modules/mod_mapcache.so' >> /etc/apache2/apache2.conf" +sudo su -c "echo '' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' ' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' Require all granted' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' ' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' MapCacheAlias /mapcache \"/tmp/mc/mapcache.xml\"' >> /etc/apache2/apache2.conf" +sudo su -c "echo '' >> /etc/apache2/apache2.conf" + +sudo service apache2 restart diff -Nru mapcache-1.2.1/.travis.yml mapcache-1.6.1/.travis.yml --- mapcache-1.2.1/.travis.yml 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/.travis.yml 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,56 @@ +matrix: + fast_finish: true + include: + - os: linux + dist: trusty + language: c + sudo: required + env: + - DISTRO=trusty + - BUILD_TYPE=maximum + + - os: linux + dist: trusty + language: c + sudo: required + env: + - DISTRO=trusty + - BUILD_TYPE=minimum + + - os: linux + language: c + sudo: required + env: + - DISTRO=precise + - BUILD_TYPE=maximum + +language: c + +before_install: + - sudo mv /etc/apt/sources.list.d/pgdg* /tmp + - sudo apt-get purge -y libgdal* libgeos* libspatialite* + - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + - sudo apt-get update + - sudo apt-get install cmake libspatialite-dev libfcgi-dev libproj-dev libgeos-dev libgdal-dev libtiff-dev libgeotiff-dev apache2-dev libpcre3-dev libsqlite3-dev libdb-dev +# For testing + - sudo apt-get install libxml2-utils apache2 gdal-bin + +script: + - mkdir build + - cd build + - if test "$BUILD_TYPE" = "maximum"; then cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DWITH_TIFF=ON -DWITH_GEOTIFF=ON -DWITH_TIFF_WRITE_SUPPORT=ON -DWITH_PCRE=ON -DWITH_SQLITE=ON -DWITH_BERKELEY_DB=ON; else cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DWITH_TIFF=OFF -DWITH_GEOTIFF=OFF -DWITH_TIFF_WRITE_SUPPORT=OFF -DWITH_PCRE=OFF -DWITH_SQLITE=OFF -DWITH_BERKELEY_DB=OFF -DWITH_GDAL=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CGI=OFF -DWITH_APACHE=OFF -DWITH_OGR=OFF -DWITH_MAPSERVER=OFF; fi + - make -j3 + - sudo make install +# Only test with Apache 2.4 + - if test "$DISTRO" = "trusty" -a "$BUILD_TYPE" = "maximum"; then cd ../tests; sh ./travis_setup.sh; sh ./run_tests.sh; fi + + +#notifications: +# email: +# recipients: +# - thomas.bonfort@gmail.com +# irc: +# channels: +# - "irc.freenode.org#mapserver" +# use_notice: true + diff -Nru mapcache-1.2.1/util/mapcache_seed.c mapcache-1.6.1/util/mapcache_seed.c --- mapcache-1.2.1/util/mapcache_seed.c 2014-01-03 11:35:58.000000000 +0000 +++ mapcache-1.6.1/util/mapcache_seed.c 2017-09-29 21:31:13.000000000 +0000 @@ -37,6 +37,7 @@ #include #ifndef _WIN32 #include +#include #define USE_FORK #include #endif @@ -53,6 +54,7 @@ #include apr_queue_t *work_queue; +apr_queue_t *log_queue; #if defined(USE_OGR) && defined(USE_GEOS) #define USE_CLIPPERS @@ -69,7 +71,7 @@ mapcache_tileset *tileset_transfer; mapcache_cfg *cfg; mapcache_context ctx; -apr_table_t *dimensions=NULL; +apr_array_header_t *dimensions=NULL; int minzoom=-1; int maxzoom=-1; mapcache_grid_link *grid_link; @@ -80,23 +82,30 @@ int force = 0; int sig_int_received = 0; int error_detected = 0; +double percent_failed_allowed = 1.0; +int n_metatiles_tot = 0; +int n_nodata_tot = 0; +int rate_limit = 0; +FILE *failed_log = NULL, *retry_log = NULL; +#define FAIL_BACKLOG_COUNT 1000 apr_time_t age_limit = 0; -int seededtilestot=0, seededtiles=0, queuedtilestot=0; -struct mctimeval lastlogtime,starttime; +struct mctimeval starttime; typedef enum { MAPCACHE_CMD_SEED, MAPCACHE_CMD_STOP, MAPCACHE_CMD_DELETE, MAPCACHE_CMD_SKIP, - MAPCACHE_CMD_TRANSFER + MAPCACHE_CMD_TRANSFER, + MAPCACHE_CMD_STOP_RECURSION } cmd; typedef enum { MAPCACHE_ITERATION_UNSET, MAPCACHE_ITERATION_DEPTH_FIRST, - MAPCACHE_ITERATION_LEVEL_FIRST + MAPCACHE_ITERATION_LEVEL_FIRST, + MAPCACHE_ITERATION_LOG } mapcache_iteration_mode; mapcache_iteration_mode iteration_mode = MAPCACHE_ITERATION_UNSET; @@ -108,6 +117,19 @@ int z; }; +typedef enum { + MAPCACHE_STATUS_OK, + MAPCACHE_STATUS_FAIL, + MAPCACHE_STATUS_FINISHED +} s_status; + +struct seed_status { + s_status status; + int x,y,z; + int nodata; + char *msg; +}; + #ifdef USE_FORK struct msg_cmd { long mtype; @@ -119,6 +141,9 @@ int push_queue(struct seed_cmd cmd) { + struct seed_cmd *pcmd; + int retries=0; + int ret; #ifdef USE_FORK if(nprocesses > 1) { struct msg_cmd mcmd; @@ -131,14 +156,23 @@ return APR_SUCCESS; } #endif - struct seed_cmd *pcmd = calloc(1,sizeof(struct seed_cmd)); + pcmd = calloc(1,sizeof(struct seed_cmd)); *pcmd = cmd; - return apr_queue_push(work_queue,pcmd); + ret = apr_queue_push(work_queue,pcmd); + while (ret == APR_EINTR && retries < 10) { + retries++; + ret = apr_queue_push(work_queue,pcmd); + } + if(ret == APR_EINTR) { + printf("failed to push tile %d %d %d after 10 retries\n",cmd.z,cmd.y,cmd.x); + return APR_EGENERAL; + } + return ret; } int pop_queue(struct seed_cmd *cmd) { - int ret; + int ret,retries=0; struct seed_cmd *pcmd; #ifdef USE_FORK @@ -152,8 +186,12 @@ return APR_SUCCESS; } #endif - + ret = apr_queue_pop(work_queue, (void**)&pcmd); + while(ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_pop(work_queue, (void**)&pcmd); + } if(ret == APR_SUCCESS) { *cmd = *pcmd; free(pcmd); @@ -163,7 +201,7 @@ int trypop_queue(struct seed_cmd *cmd) { - int ret; + int ret,retries=0; struct seed_cmd *pcmd; #ifdef USE_FORK @@ -181,6 +219,10 @@ } #endif ret = apr_queue_trypop(work_queue,(void**)&pcmd); + while(ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_trypop(work_queue,(void**)&pcmd); + } if(ret == APR_SUCCESS) { *cmd = *pcmd; free(pcmd); @@ -188,9 +230,13 @@ return ret; } +#define SEEDER_OPT_THREAD_DELAY 256 +#define SEEDER_OPT_RATE_LIMIT 257 + static const apr_getopt_option_t seed_options[] = { /* long-option, short-option, has-arg flag, description */ { "config", 'c', TRUE, "configuration file (/path/to/mapcache.xml)"}, + { "cache", 'C', TRUE, "override cache used by selected tileset (useful for selectively seeding fallback/multitier caches)"}, #ifdef USE_CLIPPERS { "ogr-datasource", 'd', TRUE, "ogr datasource to get features from"}, #endif @@ -199,16 +245,19 @@ { "force", 'f', FALSE, "force tile recreation even if it already exists" }, { "grid", 'g', TRUE, "grid to seed" }, { "help", 'h', FALSE, "show help" }, - { "iteration-mode", 'i', TRUE, "either \"drill-down\" or \"level-by-level\". Default is to use drill-down for g, WGS84 and GoogleMapsCompatible grids, and level-by-level for others. Use this flag to override." }, + { "iteration-mode", 'i', TRUE, "either \"drill-down\" or \"scanline\". Default is to use drill-down for g, WGS84 and GoogleMapsCompatible grids, and scanline for others. Use this flag to override." }, #ifdef USE_CLIPPERS { "ogr-layer", 'l', TRUE, "layer inside datasource"}, #endif + { "log-failed", 'L', TRUE, "log failed tiles to [file]"}, { "mode", 'm', TRUE, "mode: seed (default), delete or transfer" }, { "metasize", 'M', TRUE, "override metatile size while seeding, eg 8,8" }, { "nthreads", 'n', TRUE, "number of parallel threads to use (incompatible with -p/--nprocesses)" }, { "older", 'o', TRUE, "reseed tiles older than supplied date (format: year/month/day hour:minute, eg: 2011/01/31 20:45" }, { "nprocesses", 'p', TRUE, "number of parallel processes to use (incompatible with -n/--nthreads)" }, + { "percent", 'P', TRUE, "percent of failed requests allowed from the last 1000 before we abort (default: 1(%), set to 0 to abort on first error)" }, { "quiet", 'q', FALSE, "don't show progress info" }, + { "retry-failed", 'R', TRUE, "retry failed requests logged to [file] by --log-failed" }, #ifdef USE_CLIPPERS { "ogr-sql", 's', TRUE, "sql to filter inside layer"}, #endif @@ -219,14 +268,19 @@ #endif { "transfer", 'x', TRUE, "tileset to transfer" }, { "zoom", 'z', TRUE, "min and max zoomlevels to seed, separated by a comma. eg 0,6" }, + { "rate-limit", SEEDER_OPT_RATE_LIMIT, TRUE, "maximum number of tiles/second to seed"}, + { "thread-delay", SEEDER_OPT_THREAD_DELAY, TRUE, "delay in seconds between rendering thread creation (ramp up)"}, { NULL, 0, 0, NULL } }; void handle_sig_int(int signal) { +#define signal_msg "SIGINT received, waiting for threads to finish\npress ctrl-C again to force terminate\n" if(!sig_int_received) { - fprintf(stderr,"SIGINT received, waiting for threads to finish\n"); - fprintf(stderr,"press ctrl-C again to force terminate, you might end up with locked tiles\n"); + int err = write(2,signal_msg,strlen(signal_msg)); + if(err) { + //nothing we can really do here + } sig_int_received = 1; } else { exit(signal); @@ -235,12 +289,12 @@ void seed_log(mapcache_context *ctx, mapcache_log_level level, char *msg, ...) { - if(verbose) { + if(level >= MAPCACHE_WARN || verbose) { va_list args; va_start(args,msg); vfprintf(stderr,msg,args); va_end(args); - printf("\n"); + fprintf(stderr,"\n"); } } @@ -251,7 +305,7 @@ va_start(args,msg); vfprintf(stderr,msg,args); va_end(args); - printf("\n"); + fprintf(stderr,"\n"); } #ifdef USE_CLIPPERS @@ -259,6 +313,10 @@ { mapcache_metatile *mt = mapcache_tileset_metatile_get(ctx,tile); GEOSCoordSequence *mtbboxls = GEOSCoordSeq_create(5,2); + GEOSGeometry *mtbbox = GEOSGeom_createLinearRing(mtbboxls); + GEOSGeometry *mtbboxg = GEOSGeom_createPolygon(mtbbox,NULL,0); + int i; + int intersects = 0; GEOSCoordSeq_setX(mtbboxls,0,mt->map.extent.minx); GEOSCoordSeq_setY(mtbboxls,0,mt->map.extent.miny); GEOSCoordSeq_setX(mtbboxls,1,mt->map.extent.maxx); @@ -269,10 +327,6 @@ GEOSCoordSeq_setY(mtbboxls,3,mt->map.extent.maxy); GEOSCoordSeq_setX(mtbboxls,4,mt->map.extent.minx); GEOSCoordSeq_setY(mtbboxls,4,mt->map.extent.miny); - GEOSGeometry *mtbbox = GEOSGeom_createLinearRing(mtbboxls); - GEOSGeometry *mtbboxg = GEOSGeom_createPolygon(mtbbox,NULL,0); - int i; - int intersects = 0; for(i=0; i= 1) nworkers = nprocesses; - - sprintf(msg,"seeding tile %d %d %d",x,y,z); - if(lastmsglen) { - char erasestring[1024]; - int len = MAPCACHE_MIN(1023,lastmsglen); - memset(erasestring,' ',len); - erasestring[len+1]='\0'; - sprintf(erasestring,"\r%%%ds\r",lastmsglen); - printf(erasestring," "); - } - lastmsglen = strlen(msg); - printf("%s",msg); - fflush(NULL); - return; + int action = MAPCACHE_CMD_SKIP; + int tile_exists; - if(queuedtilestot>nworkers) { - struct mctimeval now_t; - float duration; - float totalduration; - seededtilestot = queuedtilestot - nworkers; +#ifdef USE_CLIPPERS + /* check we are in the requested features before checking the tile */ + if(nClippers > 0 && ogr_features_intersect_tile(ctx,tile) == 0) + return MAPCACHE_CMD_STOP_RECURSION; +#endif - mapcache_gettimeofday(&now_t,NULL); - duration = ((now_t.tv_sec-lastlogtime.tv_sec)*1000000+(now_t.tv_usec-lastlogtime.tv_usec))/1000000.0; - totalduration = ((now_t.tv_sec-starttime.tv_sec)*1000000+(now_t.tv_usec-starttime.tv_usec))/1000000.0; - if(duration>=5) { - int Nx, Ny, Ntot, Ncur, ntilessincelast; - Nx = (grid_link->grid_limits[z].maxx-grid_link->grid_limits[z].minx)/tileset->metasize_x; - Ny = (grid_link->grid_limits[z].maxy-grid_link->grid_limits[z].miny)/tileset->metasize_y; - Ntot = Nx*Ny; - Ncur = ((y-grid_link->grid_limits[z].miny)/tileset->metasize_y ) * Nx + - (x-grid_link->grid_limits[z].minx+1)/tileset->metasize_x; - ntilessincelast = seededtilestot-seededtiles; - sprintf(msg,"seeding level %d [%d/%d]: %f metatiles/sec (avg since start: %f)",z,Ncur,Ntot,ntilessincelast/duration, - seededtilestot/totalduration); - lastlogtime=now_t; - seededtiles=seededtilestot; + if(mode != MAPCACHE_CMD_TRANSFER && force) { + if(mode == MAPCACHE_CMD_DELETE) { + tile_exists = 1; } else { - return; + tile_exists = 0; } } else { - sprintf(msg,"seeding level %d",z); - + int i; + if(tile->tileset->dimension_assembly_type != MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim->cached_value = rdim->requested_value; + } + } else { + if(tile->dimensions) { + mapcache_extent extent; + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + for(i=0; idimensions->nelts; i++) { + apr_array_header_t *rdim_vals; + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim_vals = rdim->dimension->get_entries_for_value(ctx,rdim->dimension,rdim->requested_value, tile->tileset, NULL, tile->grid_link->grid); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_CMD_SKIP; + } + if(rdim_vals->nelts > 1) { + ctx->set_error(ctx,500,"dimension (%s) for tileset (%s) returned invalid number of subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return MAPCACHE_CMD_SKIP; + } + if(rdim_vals->nelts == 0) { + ctx->set_error(ctx,404,"dimension (%s) for tileset (%s) returned no subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return MAPCACHE_CMD_SKIP; + } + rdim->cached_value = APR_ARRAY_IDX(rdim_vals,0,char*); + } + } + } + tile_exists = mapcache_cache_tile_exists(ctx,tileset->_cache,tile); } - if(lastmsglen) { - char erasestring[1024]; - int len = MAPCACHE_MIN(1023,lastmsglen); - memset(erasestring,' ',len); - erasestring[len+1]='\0'; - sprintf(erasestring,"\r%%%ds\r",lastmsglen); - printf(erasestring," "); - } - lastmsglen = strlen(msg); - printf("%s",msg); - fflush(NULL); -} - -cmd examine_tile(mapcache_context *ctx, mapcache_tile *tile) -{ - int action = MAPCACHE_CMD_SKIP; - int intersects = -1; - int tile_exists = force?0:tileset->cache->tile_exists(ctx,tile); /* if the tile exists and a time limit was specified, check the tile modification date */ if(tile_exists) { if(age_limit) { - if(tileset->cache->tile_get(ctx,tile) == MAPCACHE_SUCCESS) { + if(mapcache_cache_tile_get(ctx,tileset->_cache, tile) == MAPCACHE_SUCCESS) { if(tile->mtime && tile->mtime 0) { - intersects = ogr_features_intersect_tile(ctx,tile); - } -#endif - if(intersects != 0) { - /* the tile intersects the ogr features, or there was no clipping asked for: seed it */ - if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { - mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); - /* if we are in mode transfer, delete it from the dst tileset */ - if (mode == MAPCACHE_CMD_TRANSFER) { - tile->tileset = tileset_transfer; - if (tileset_transfer->cache->tile_exists(ctx,tile)) { - mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); - } - tile->tileset = tileset; + if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { + mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); + /* if we are in mode transfer, delete it from the dst tileset */ + if (mode == MAPCACHE_CMD_TRANSFER) { + tile->tileset = tileset_transfer; + if (mapcache_cache_tile_exists(ctx,tile->tileset->_cache, tile)) { + mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); } - action = mode; - } else { //if(action == MAPCACHE_CMD_DELETE) - action = MAPCACHE_CMD_DELETE; + tile->tileset = tileset; } - } else { - /* the tile does not intersect the ogr features, and already exists, do nothing */ - action = MAPCACHE_CMD_SKIP; + action = mode; + } else { //if(action == MAPCACHE_CMD_DELETE) + action = MAPCACHE_CMD_DELETE; } } } else { @@ -401,7 +423,7 @@ /* the tile exists in the source tileset, check if the tile exists in the destination cache */ tile->tileset = tileset_transfer; - if (tileset_transfer->cache->tile_exists(ctx,tile)) { + if (!force && mapcache_cache_tile_exists(ctx,tile->tileset->_cache, tile)) { action = MAPCACHE_CMD_SKIP; } else { action = MAPCACHE_CMD_TRANSFER; @@ -414,21 +436,8 @@ } } else { // the tile does not exist - if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { -#ifdef USE_CLIPPERS - /* check we are in the requested features before deleting the tile */ - if(nClippers > 0) { - if(ogr_features_intersect_tile(ctx,tile)) { - action = mode; - } else { - action = MAPCACHE_CMD_SKIP; - } - } else { - action = mode; - } -#else + if (mode == MAPCACHE_CMD_SEED) { action = mode; -#endif } else { action = MAPCACHE_CMD_SKIP; } @@ -437,6 +446,27 @@ return action; } +double rate_limit_last_time = 0; +double rate_limit_delay = 0.0; + +void rate_limit_sleep() { + if(rate_limit > 0) { + struct mctimeval now; + double now_time; + mapcache_gettimeofday(&now,NULL); + now_time = now.tv_sec + now.tv_usec / 1000000.0; + + if((now_time - rate_limit_last_time) < rate_limit_delay) { + apr_sleep((int)((rate_limit_delay - (now_time - rate_limit_last_time)) * 1000000)); + /*last_time = now_time + delay - (now_time - last_time); //now plus the time we slept */ + rate_limit_last_time = rate_limit_delay + rate_limit_last_time; + } else { + rate_limit_last_time = now_time; + } + + } +} + void cmd_recurse(mapcache_context *cmd_ctx, mapcache_tile *tile) { cmd action; @@ -450,9 +480,7 @@ if(sig_int_received || error_detected) { //stop if we were asked to stop by hitting ctrl-c //remove all items from the queue struct seed_cmd entry; - while (trypop_queue(&entry)!=APR_EAGAIN) { - queuedtilestot--; - } + while (trypop_queue(&entry)!=APR_EAGAIN) /*do nothing*/; return; } @@ -465,10 +493,13 @@ cmd.y = tile->y; cmd.z = tile->z; cmd.command = action; + if(rate_limit > 0) + rate_limit_sleep(); push_queue(cmd); - queuedtilestot++; - progresslog(tile->x,tile->y,tile->z); } + + if(action == MAPCACHE_CMD_STOP_RECURSION) + return; //recurse into our 4 child metatiles @@ -487,9 +518,9 @@ */ - mapcache_grid_get_extent(cmd_ctx, grid_link->grid, + mapcache_grid_get_tile_extent(cmd_ctx, grid_link->grid, curx, cury, curz, &bboxbl); - mapcache_grid_get_extent(cmd_ctx, grid_link->grid, + mapcache_grid_get_tile_extent(cmd_ctx, grid_link->grid, curx+tileset->metasize_x-1, cury+tileset->metasize_y-1, curz, &bboxtr); epsilon = (bboxbl.maxx-bboxbl.minx)*0.01; mapcache_grid_get_xy(cmd_ctx,grid_link->grid, @@ -521,7 +552,7 @@ tile->z = curz; } -void cmd_worker() +void feed_worker() { int n; mapcache_tile *tile; @@ -533,7 +564,11 @@ if(nprocesses >= 1) nworkers = nprocesses; apr_pool_create(&cmd_ctx.pool,ctx.pool); tile = mapcache_tileset_tile_create(ctx.pool, tileset, grid_link); - tile->dimensions = dimensions; + tile->dimensions = mapcache_requested_dimensions_clone(ctx.pool,dimensions); + if(rate_limit > 0) { + /* compute time between seed commands accounting for max rate-limit and current metasize */ + rate_limit_delay = (tileset->metasize_x * tileset->metasize_y) / (double)rate_limit; + } if(iteration_mode == MAPCACHE_ITERATION_DEPTH_FIRST) { do { tile->x = x; @@ -559,11 +594,16 @@ if(sig_int_received || error_detected) { //stop if we were asked to stop by hitting ctrl-c //remove all items from the queue struct seed_cmd entry; - while (trypop_queue(&entry)!=APR_EAGAIN) { - queuedtilestot--; - } + while (trypop_queue(&entry)!=APR_EAGAIN) /* do nothing */; break; } + if(iteration_mode == MAPCACHE_ITERATION_LOG) { + if(3 != fscanf(retry_log,"%d,%d,%d\n",&x,&y,&z)) { + break; + } else { + printf("from log: %d %d %d\n",x,y,z); + } + } tile->x = x; tile->y = y; tile->z = z; @@ -576,9 +616,9 @@ cmd.y = y; cmd.z = z; cmd.command = action; + if(rate_limit > 0) + rate_limit_sleep(); push_queue(cmd); - queuedtilestot++; - progresslog(x,y,z); } //compute next x,y,z @@ -603,10 +643,6 @@ cmd.command = MAPCACHE_CMD_STOP; push_queue(cmd); } - - if(error_detected && ctx.get_error_message(&ctx)) { - printf("%s\n",ctx.get_error_message(&ctx)); - } } @@ -619,7 +655,9 @@ apr_pool_create(&seed_ctx.pool,ctx.pool); apr_pool_create(&tpool,ctx.pool); tile = mapcache_tileset_tile_create(tpool, tileset, grid_link); - tile->dimensions = dimensions; + if(dimensions) { + tile->dimensions = mapcache_requested_dimensions_clone(tpool,dimensions); + } while(1) { struct seed_cmd cmd; apr_status_t ret; @@ -630,30 +668,84 @@ tile->x = cmd.x; tile->y = cmd.y; tile->z = cmd.z; + tile->nodata = 0; + tile->encoded_data = NULL; + tile->raw_image = NULL; + if(tile->dimensions) { + int i; + if(tileset->dimension_assembly_type == MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + mapcache_extent extent; + mapcache_grid_get_tile_extent(&seed_ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + for(i=0; idimensions->nelts; i++) { + apr_array_header_t *rdim_vals; + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim_vals = rdim->dimension->get_entries_for_value(&seed_ctx,rdim->dimension,rdim->requested_value, tile->tileset, NULL, tile->grid_link->grid); + GC_CHECK_ERROR(&seed_ctx); + if(rdim_vals->nelts > 1) { + seed_ctx.set_error(&seed_ctx,500,"dimension (%s) for tileset (%s) returned invalid number of subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return; + } + if(rdim_vals->nelts == 0) { + seed_ctx.set_error(&seed_ctx,404,"dimension (%s) for tileset (%s) returned no subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return; + } + rdim->cached_value = APR_ARRAY_IDX(rdim_vals,0,char*); + } + } else { + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim->cached_value = NULL; + } + } + } if(cmd.command == MAPCACHE_CMD_SEED) { - /* aquire a lock on the metatile ?*/ - mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); - int isLocked = mapcache_lock_or_wait_for_resource(&seed_ctx, mapcache_tileset_metatile_resource_key(&seed_ctx,mt)); - if(isLocked == MAPCACHE_TRUE) { + if(!tile->dimensions || tileset->dimension_assembly_type == MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); /* this will query the source to create the tiles, and save them to the cache */ mapcache_tileset_render_metatile(&seed_ctx, mt); - mapcache_unlock_resource(&seed_ctx, mapcache_tileset_metatile_resource_key(&seed_ctx,mt)); + } else { + mapcache_tileset_tile_set_get_with_subdimensions(&seed_ctx,tile); } } else if (cmd.command == MAPCACHE_CMD_TRANSFER) { - int i; - mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); - for (i = 0; i < mt->ntiles; i++) { - mapcache_tile *subtile = &mt->tiles[i]; - mapcache_tileset_tile_get(&seed_ctx, subtile); - subtile->tileset = tileset_transfer; - tileset_transfer->cache->tile_set(&seed_ctx, subtile); + mapcache_tileset_tile_get(&seed_ctx, tile); + if(!tile->nodata && !GC_HAS_ERROR(&seed_ctx)) { + mapcache_tileset *tmp_tileset = tile->tileset; + tile->tileset = tileset_transfer; + mapcache_cache_tile_set(&seed_ctx, tile->tileset->_cache, tile); + tile->tileset = tmp_tileset; } } else { //CMD_DELETE mapcache_tileset_tile_delete(&seed_ctx,tile,MAPCACHE_TRUE); } - if(seed_ctx.get_error(&seed_ctx)) { - error_detected++; - ctx.log(&ctx,MAPCACHE_INFO,seed_ctx.get_error_message(&seed_ctx)); + + { + struct seed_status *st = calloc(1,sizeof(struct seed_status)); + int retries=0; + st->x=tile->x; + st->y=tile->y; + st->z=tile->z; + st->nodata = tile->nodata; + if(seed_ctx.get_error(&seed_ctx)) { + st->status = MAPCACHE_STATUS_FAIL; + st->msg = strdup(seed_ctx.get_error_message(&seed_ctx)); + seed_ctx.clear_errors(&seed_ctx); + } else { + st->status = MAPCACHE_STATUS_OK; + } + ret = apr_queue_push(log_queue,(void*)st); + while( ret == APR_EINTR && retries < 10) { + retries++; + ret = apr_queue_push(log_queue,(void*)st); + } + if( ret == APR_EINTR) { + printf("FATAL ERROR: unable to log progress after 10 retries, aborting\n"); + break; + } + if(ret != APR_SUCCESS) + { + printf("FATAL ERROR: unable to log progress\n"); + break; + } } } } @@ -664,11 +756,85 @@ return 0; } #endif + static void* APR_THREAD_FUNC seed_thread(apr_thread_t *thread, void *data) { seed_worker(); return NULL; } +static void* APR_THREAD_FUNC feed_thread_fn(apr_thread_t *thread, void *data) { + //start the thread that will populate the queue. + feed_worker(); + return NULL; +} + +static void* APR_THREAD_FUNC log_thread_fn(apr_thread_t *thread, void *data) { + size_t cur; + double last_time; + double now_time; + int i; + int nfailed; + int ntotal; + double pct; + char failed[FAIL_BACKLOG_COUNT]; + memset(failed,-1,FAIL_BACKLOG_COUNT); + cur=0; + last_time=0; + while(1) { + int retries = 0; + struct seed_status *st; + apr_status_t ret = apr_queue_pop(log_queue, (void**)&st); + while(ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_pop(log_queue, (void**)&st); + } + if(ret != APR_SUCCESS || !st) break; + if(st->status == MAPCACHE_STATUS_FINISHED) + return NULL; + if(st->status == MAPCACHE_STATUS_OK) { + failed[cur]=0; + n_metatiles_tot++; + if(st->nodata) { + n_nodata_tot++; + } + if(!quiet) { + struct mctimeval now; + mapcache_gettimeofday(&now,NULL); + now_time = now.tv_sec + now.tv_usec / 1000000.0; + if((now_time - last_time) > 1.0) { + printf(" \r"); + printf("seeded %d tiles, now at z%d x%d y%d\r",n_metatiles_tot*tileset->metasize_x*tileset->metasize_y, st->z,st->x,st->y); + fflush(stdout); + last_time = now_time; + } + } + } else { + /* count how many errors and successes we have */ + failed[cur]=1; + nfailed=0; + ntotal=0; + if(failed_log) { + fprintf(failed_log,"%d,%d,%d\n",st->x,st->y,st->z); + } + for(i=0; i=0) ntotal++; + if(failed[i]==1) nfailed++; + } + ctx.log(&ctx, MAPCACHE_WARN, "failed to seed tile z%d,x%d,y%d:\n%s\n", st->z,st->x,st->y,st->msg); + pct = ((double)nfailed / (double)ntotal) * 100; + if(pct > percent_failed_allowed) { + ctx.log(&ctx, MAPCACHE_ERROR, "aborting seed as %.1f%% of the last %d requests failed\n", pct, FAIL_BACKLOG_COUNT); + error_detected = 1; + } + } + if(st->msg) free(st->msg); + free(st); + cur++; + cur %= FAIL_BACKLOG_COUNT; + } + return NULL; +} + void notice(const char *fmt, ...) { @@ -712,10 +878,18 @@ printf("usage: %s options\n",progname); while(seed_options[i].name) { - if(seed_options[i].has_arg==TRUE) { - printf("-%c|--%s [value]: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + if(seed_options[i].optch<256) { + if(seed_options[i].has_arg==TRUE) { + printf("-%c|--%s [value]: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + } else { + printf("-%c|--%s: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + } } else { - printf("-%c|--%s: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + if(seed_options[i].has_arg==TRUE) { + printf(" --%s [value]: %s\n",seed_options[i].name, seed_options[i].description); + } else { + printf(" --%s: %s\n",seed_options[i].name, seed_options[i].description); + } } i++; } @@ -733,11 +907,12 @@ /* initialize apr_getopt_t */ apr_getopt_t *opt; const char *configfile=NULL; - apr_thread_t **threads; - apr_threadattr_t *thread_attrs; + apr_thread_t **seed_threads; + apr_thread_t *log_thread,*feed_thread; const char *tileset_name=NULL; const char *tileset_transfer_name=NULL; const char *grid_name = NULL; + const char *cache_override = NULL; int *zooms = NULL;//[2]; mapcache_extent *extent = NULL;//[4]; int optch; @@ -750,8 +925,11 @@ int *metasizes = NULL;//[2]; int metax=-1,metay=-1; double *extent_array = NULL; + double thread_delay = 0.0; #ifdef USE_CLIPPERS + OGRFeatureH hFeature; + GEOSWKTReader *geoswktreader; const char *ogr_where = NULL; const char *ogr_layer = NULL; const char *ogr_sql = NULL; @@ -762,15 +940,12 @@ (void) signal(SIGINT,handle_sig_int); apr_pool_create(&ctx.pool,NULL); mapcache_context_init(&ctx); - ctx.process_pool = ctx.pool; cfg = mapcache_configuration_create(ctx.pool); ctx.config = cfg; ctx.log= mapcache_context_seeding_log; apr_getopt_init(&opt, ctx.pool, argc, argv); - seededtiles=seededtilestot=queuedtilestot=0; mapcache_gettimeofday(&starttime,NULL); - lastlogtime=starttime; argdimensions = apr_table_make(ctx.pool,3); @@ -792,6 +967,9 @@ case 'c': configfile = optarg; break; + case 'C': + cache_override = optarg; + break; case 'g': grid_name = optarg; break; @@ -804,10 +982,22 @@ case 'i': if(!strcmp(optarg,"drill-down")) { iteration_mode = MAPCACHE_ITERATION_DEPTH_FIRST; - } else if(!strcmp(optarg,"level-by-level")) { + } else if(!strcmp(optarg,"level-by-level") || !strcmp(optarg, "scanline")) { iteration_mode = MAPCACHE_ITERATION_LEVEL_FIRST; } else { - return usage(argv[0],"invalid iteration mode, expecting \"drill-down\" or \"level-by-level\""); + return usage(argv[0],"invalid iteration mode, expecting \"drill-down\" or \"scanline\""); + } + break; + case 'L': + failed_log = fopen(optarg,"w"); + if(!failed_log) { + return usage(argv[0],"failed to open -L|--log-failed file for writing"); + } + break; + case 'R': + retry_log = fopen(optarg,"r"); + if(!retry_log) { + return usage(argv[0],"failed to open -R|--retry_failed file for writing"); } break; case 'm': @@ -835,6 +1025,12 @@ #else return usage(argv[0], "multi process seeding not available on this platform"); #endif + case 'P': + percent_failed_allowed = (double)strtol(optarg, NULL, 10); + if(percent_failed_allowed<0 || percent_failed_allowed>100 ) + return usage(argv[0], "failed to parse percent, expecting number between 0 and 100"); + break; + case 'e': if ( MAPCACHE_SUCCESS != mapcache_util_extract_double_list(&ctx, (char*)optarg, ",", &extent_array, &n) || n != 4 || extent_array[0] >= extent_array[2] || extent_array[1] >= extent_array[3] ) { @@ -884,6 +1080,16 @@ } apr_table_set(argdimensions,dimkey,dimvalue); break; + case SEEDER_OPT_THREAD_DELAY: + thread_delay = strtod(optarg, NULL); + if(thread_delay < 0.0 ) + return usage(argv[0], "failed to parse thread-delay, expecting positive number of seconds"); + break; + case SEEDER_OPT_RATE_LIMIT: + rate_limit = (int)strtol(optarg, NULL, 10); + if(rate_limit <= 0 ) + return usage(argv[0], "failed to parse rate-limit, expecting positive number of tiles per seconds"); + break; #ifdef USE_CLIPPERS case 'd': ogr_datasource = optarg; @@ -914,6 +1120,7 @@ mapcache_configuration_post_config(&ctx,cfg); if(ctx.get_error(&ctx)) return usage(argv[0],ctx.get_error_message(&ctx)); + mapcache_connection_pool_create(&ctx.connection_pool, ctx.pool); } #ifdef USE_CLIPPERS @@ -926,6 +1133,7 @@ } if(ogr_datasource) { + int f=0; OGRDataSourceH hDS = NULL; OGRLayerH layer = NULL; OGRRegisterAll(); @@ -973,21 +1181,20 @@ clippers = (const GEOSPreparedGeometry**)malloc(nClippers*sizeof(GEOSPreparedGeometry*)); - OGRFeatureH hFeature; - GEOSWKTReader *geoswktreader = GEOSWKTReader_create(); + geoswktreader = GEOSWKTReader_create(); OGR_L_ResetReading(layer); - extent = apr_palloc(ctx.pool,4*sizeof(mapcache_extent)); - int f=0; + extent = apr_palloc(ctx.pool,sizeof(mapcache_extent)); while( (hFeature = OGR_L_GetNextFeature(layer)) != NULL ) { + char *wkt; + GEOSGeometry *geosgeom; + OGREnvelope ogr_extent; OGRGeometryH geom = OGR_F_GetGeometryRef(hFeature); if(!geom || !OGR_G_IsValid(geom)) continue; - char *wkt; OGR_G_ExportToWkt(geom,&wkt); - GEOSGeometry *geosgeom = GEOSWKTReader_read(geoswktreader,wkt); + geosgeom = GEOSWKTReader_read(geoswktreader,wkt); free(wkt); clippers[f] = GEOSPrepare(geosgeom); //GEOSGeom_destroy(geosgeom); - OGREnvelope ogr_extent; OGR_G_GetEnvelope (geom, &ogr_extent); if(f == 0) { extent->minx = ogr_extent.MinX; @@ -1017,10 +1224,16 @@ if(!tileset) { return usage(argv[0], "tileset not found in configuration"); } - if(tileset->read_only) { - printf("tileset %s is read-only, switching it to read-write for seeding\n",tileset_name); + if(tileset->read_only && mode != MAPCACHE_CMD_TRANSFER) { + printf("tileset (%s) is read-only, switching it to read-write for seeding\n",tileset_name); tileset->read_only = 0; } + if( mode == MAPCACHE_CMD_TRANSFER ) { + if( tileset->read_only == 0 ) { + printf("switching tileset (%s) to read-only as we are in transfer mode\n",tileset_name); + } + tileset->read_only = 1; + } if( ! grid_name ) { grid_link = APR_ARRAY_IDX(tileset->grid_links,0,mapcache_grid_link*); } else { @@ -1036,6 +1249,38 @@ return usage(argv[0],"grid not configured for tileset"); } } +#ifdef USE_CLIPPERS + if(ogr_datasource) { + /* check that the provided ogr features are compatible with the grid units */ + if(grid_link->grid->unit == MAPCACHE_UNIT_DEGREES) { + if(extent->minx < -181.0 || + extent->maxx > 181.0 || + extent->miny < -91.0 || + extent->maxy > 91.0) { + printf("\n********************************************************************************\n" + "* WARNING!!!: you are seeding a grid in latlon degreees,\n" + "* but your provided OGR intersection features span (%f,%f,%f,%f).\n" + "* this seems like an error, you should be providing OGR features that\n" + "* are in the same projection as the grid you want to seed\n" + "********************************************************************************\n\n", + extent->minx, extent->miny, extent->maxx, extent->maxy); + } + } else { + if(extent->minx > -181.0 && + extent->maxx < 181.0 && + extent->miny > -91.0 && + extent->maxy < 91.0) { + printf("\n********************************************************************************\n" + "* WARNING!!!: you are seeding a grid that is not in latlon degreees,\n" + "* but your provided OGR intersection features span (%f,%f,%f,%f) which seem to be in degrees.\n" + "* this seems like an error, you should be providing OGR features that\n" + "* are in the same projection as the grid you want to seed\n" + "********************************************************************************\n\n", + extent->minx, extent->miny, extent->maxx, extent->maxy); + } + } + } +#endif if(iteration_mode == MAPCACHE_ITERATION_UNSET) { if(!strcmp(grid_link->grid->name,"g") || !strcmp(grid_link->grid->name,"WGS84") || !strcmp(grid_link->grid->name,"GoogleMapsCompatible")) { @@ -1044,6 +1289,10 @@ iteration_mode = MAPCACHE_ITERATION_LEVEL_FIRST; } } + if(retry_log) { + iteration_mode = MAPCACHE_ITERATION_LOG; + } + if(minzoom == -1 && maxzoom == -1) { minzoom = grid_link->minz; maxzoom = grid_link->maxz - 1; @@ -1051,7 +1300,7 @@ if(minzoomminz) minzoom = grid_link->minz; if(maxzoom>= grid_link->maxz) maxzoom = grid_link->maxz - 1; if(grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && maxzoom > grid_link->max_cached_zoom) { - maxzoom = grid_link->max_cached_zoom; + return usage(argv[0],"requested maxzoom %d is higher than configured max-cached-zoom %d for grid",maxzoom,grid_link->max_cached_zoom); } /* adjust metasize */ @@ -1063,14 +1312,23 @@ /* ensure our metasize is a power of 2 in drill down mode */ if(iteration_mode == MAPCACHE_ITERATION_DEPTH_FIRST) { if(!isPowerOfTwo(tileset->metasize_x) || !isPowerOfTwo(tileset->metasize_y)) { - return usage(argv[0],"metatile size is not set to a power of two and iteration mode set to \"drill-down\", rerun with e.g -M 8,8, or force iteration mode to \"level-by-level\""); + return usage(argv[0],"metatile size is not set to a power of two and iteration mode set to \"drill-down\", rerun with e.g -M 8,8, or force iteration mode to \"scanline\""); } } + if(cache_override) { + mapcache_cache *co = mapcache_configuration_get_cache(cfg, cache_override); + if(!co) { + return usage(argv[0], "overridden cache\"%s\" not found in configuration", cache_override); + } else { + tileset->_cache = co; + } + } } if (mode == MAPCACHE_CMD_TRANSFER) { + tileset->metasize_x = tileset->metasize_y = 1; if (!tileset_transfer_name) return usage(argv[0],"tileset where tiles should be transferred to not specified"); @@ -1120,64 +1378,55 @@ } /* validate the supplied dimensions */ - if (!apr_is_empty_array(tileset->dimensions) || tileset->timedimension) { + if (!apr_is_empty_array(tileset->dimensions)) { int i; const char *value; - dimensions = apr_table_make(ctx.pool,3); - if (!apr_is_empty_array(tileset->dimensions)) { - for(i=0; idimensions->nelts; i++) { - mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - if((value = (char*)apr_table_get(argdimensions,dimension->name)) != NULL) { - char *tmpval = apr_pstrdup(ctx.pool,value); - int ok = dimension->validate(&ctx,dimension,&tmpval); - if(GC_HAS_ERROR(&ctx) || ok != MAPCACHE_SUCCESS ) { - return usage(argv[0],"failed to validate dimension"); - return 1; - } else { - /* validate may have changed the dimension value, so set that value into the dimensions table */ - apr_table_setn(dimensions,dimension->name,tmpval); - } - } else { - /* a dimension was not specified on the command line, add the default value */ - apr_table_setn(dimensions, dimension->name, dimension->default_value); - } + dimensions = apr_array_make(ctx.pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); + for(i=0; idimensions->nelts; i++) { + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); + mapcache_requested_dimension *rdim = apr_pcalloc(ctx.pool,sizeof(mapcache_requested_dimension)); + rdim->dimension = dimension; + if((value = apr_table_get(argdimensions,dimension->name)) == NULL) { + value = dimension->default_value; + } + rdim->requested_value = apr_pstrdup(ctx.pool,value); + APR_ARRAY_PUSH(dimensions,mapcache_requested_dimension*)=rdim; + } + if(tileset->dimension_assembly_type != MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + if(!tileset->store_dimension_assemblies) { + return usage(argv[0],"cannot seed a layer with dimension assembling and no caching of resulting assembly"); } - } - if(tileset->timedimension) { - if((value = (char*)apr_table_get(argdimensions,tileset->timedimension->key)) != NULL) { - apr_array_header_t *timedim_selected = mapcache_timedimension_get_entries_for_value(&ctx,tileset->timedimension, tileset, grid_link->grid, extent, value); - if(GC_HAS_ERROR(&ctx) || !timedim_selected) { - return usage(argv[0],"failed to validate time dimension"); - } - if(timedim_selected->nelts == 0) { - return usage(argv[0],"Time dimension %s=%s returns no configured entry",tileset->timedimension->key, - value); - } - if(timedim_selected->nelts > 1) { - return usage(argv[0],"Time dimension %s=%s returns more than 1 configured entries",tileset->timedimension->key, - value); - } - apr_table_set(dimensions,tileset->timedimension->key,APR_ARRAY_IDX(timedim_selected,0,char*)); - } else { - return usage(argv[0],"tileset references a TIME dimension, but none supplied on commandline. (hint: -D %s=",tileset->timedimension->key); - + if(tileset->metasize_x != 1 || tileset->metasize_y != 1) { + return usage(argv[0],"cannot seed a layer with dimension assembling and metatiling enabled (hint: rerun with -M 1,1 to disable metatiling"); } } } + if(nthreads == 0 && nprocesses == 0) { nthreads = 1; } if(nthreads >= 1 && nprocesses >= 1) { return usage(argv[0],"cannot set both nthreads and nprocesses"); } + + { + /* start the logging thread */ + //create the queue where the seeding statuses will be put + apr_threadattr_t *log_thread_attrs; + apr_queue_create(&log_queue,MAPCACHE_MAX(nthreads,nprocesses),ctx.pool); + + //start the rendering threads. + apr_threadattr_create(&log_thread_attrs, ctx.pool); + apr_thread_create(&log_thread, log_thread_attrs, log_thread_fn, NULL, ctx.pool); + } + if(nprocesses > 1) { #ifdef USE_FORK key_t key; int i; pid_t *pids = malloc(nprocesses*sizeof(pid_t)); struct msqid_ds queue_ds; - ctx.threadlock = NULL; key = ftok(argv[0], 'B'); if ((msqid = msgget(key, 0644 | IPC_CREAT|S_IRUSR|S_IWUSR)) == -1) { return usage(argv[0],"failed to create sysv ipc message queue"); @@ -1212,7 +1461,7 @@ pids[i] = pid; } } - cmd_worker(); + feed_worker(); for(i=0; i 0) { + apr_sleep((int)(thread_delay * 1000000)); + } + apr_thread_create(&seed_threads[n], seed_thread_attrs, seed_thread, NULL, ctx.pool); } - cmd_worker(); + + + //the worker has finished generating the list of tiles to be seeded, now wait for the rendering threads to finish for(n=0; nstatus = MAPCACHE_STATUS_FINISHED; + ret = apr_queue_push(log_queue,(void*)st); + while (ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_push(log_queue,(void*)st); + } + apr_thread_join(&rv, log_thread); } - if(seededtilestot>0) { + if(n_metatiles_tot>0) { struct mctimeval now_t; float duration; + int ntilestot = n_metatiles_tot*tileset->metasize_x*tileset->metasize_y; + int nnodatatot = n_nodata_tot*tileset->metasize_x*tileset->metasize_y; mapcache_gettimeofday(&now_t,NULL); duration = ((now_t.tv_sec-starttime.tv_sec)*1000000+(now_t.tv_usec-starttime.tv_usec))/1000000.0; - printf("\nseeded %d metatiles at %g tiles/sec\n",seededtilestot, seededtilestot/duration); + + printf("\nseeded %d metatiles (%d total tiles, %d non-empty tiles) in %.1f seconds at %.1f tiles/sec (%.1f non-empty tiles/sec)\n", + n_metatiles_tot, + ntilestot, + ntilestot-nnodatatot, + duration, + ntilestot/duration, + (ntilestot-nnodatatot)/duration); + } else { + if(!error_detected) { + printf("0 tiles needed to be seeded, exiting\n"); + } } apr_terminate(); if (error_detected > 0) { - exit(1); + exit(1); } - + return 0; } /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.2.1/Vagrantfile mapcache-1.6.1/Vagrantfile --- mapcache-1.2.1/Vagrantfile 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/Vagrantfile 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,27 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +require 'socket' + +# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! +VAGRANTFILE_API_VERSION = "2" + +Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.box = "precise64" + config.vm.box_url = "http://files.vagrantup.com/precise64.box" + + config.vm.hostname = "mapcache-vagrant" + + config.vm.network :forwarded_port, guest: 80, host: 8080 + + config.vm.provider "virtualbox" do |v| + v.customize ["modifyvm", :id, "--memory", 1024, "--cpus", 2] + v.customize ["modifyvm", :id, "--ioapic", "on", "--largepages", "off", "--vtxvpid", "off"] + v.name = "mapcache-vagrant" + end + + config.vm.provision "shell", path: "scripts/vagrant/virtualbox-fix.sh" + config.vm.provision "shell", path: "scripts/vagrant/packages.sh" + config.vm.provision "shell", path: "scripts/vagrant/mapcache.sh" + +end