diff -Nru mapcache-1.4.0/apache/mod_mapcache.c mapcache-1.6.1/apache/mod_mapcache.c --- mapcache-1.4.0/apache/mod_mapcache.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/apache/mod_mapcache.c 2017-09-29 21:31:13.000000000 +0000 @@ -40,10 +40,6 @@ #include #include #include "mapcache.h" -#ifdef APR_HAS_THREADS -#include -apr_thread_mutex_t *thread_mutex = NULL; -#endif #ifndef _WIN32 #include @@ -73,6 +69,21 @@ request_rec *request; }; +typedef struct mapcache_alias_entry mapcache_alias_entry; + +struct mapcache_alias_entry { + char *endpoint; + char *configfile; + mapcache_cfg *cfg; + mapcache_connection_pool *cp; +}; + +struct mapcache_server_cfg { + apr_array_header_t *aliases; /**< list of mapcache configurations aliased to a server uri */ + apr_array_header_t *quickaliases; /**< list of mapcache configurations aliased to a server uri */ +}; +static int mapcache_alias_matches(const char *uri, const char *alias_fakename); + void apache_context_server_log(mapcache_context *c, mapcache_log_level level, char *message, ...) { mapcache_context_apache_server *ctx = (mapcache_context_apache_server*)c; @@ -166,48 +177,28 @@ return nctx; } -void init_apache_request_context(mapcache_context_apache_request *ctx) -{ - mapcache_context_init((mapcache_context*)ctx); - ctx->ctx.ctx.log = apache_context_request_log; - ctx->ctx.ctx.clone = mapcache_context_request_clone; -} - -void init_apache_server_context(mapcache_context_apache_server *ctx) +mapcache_context_apache_request* create_apache_request_context(request_rec *r) { - mapcache_context_init((mapcache_context*)ctx); - ctx->ctx.ctx.log = apache_context_server_log; -} - -static mapcache_context_apache_request* apache_request_context_create(request_rec *r) -{ - mapcache_context_apache_request *ctx = apr_pcalloc(r->pool, sizeof(mapcache_context_apache_request)); - mapcache_server_cfg *cfg = NULL; - mapcache_cfg *config = NULL; - - ctx->ctx.ctx.pool = r->pool; -#ifdef APR_HAS_THREADS - ctx->ctx.ctx.threadlock = thread_mutex; -#endif - - /* lookup the configuration object given the configuration file name */ - cfg = ap_get_module_config(r->server->module_config, &mapcache_module); - config = apr_hash_get(cfg->aliases,(void*)r->filename,APR_HASH_KEY_STRING); - ctx->ctx.ctx.config = config; - ctx->request = r; - ctx->ctx.ctx.connection_pool = cfg->cp; - init_apache_request_context(ctx); - return ctx; -} - -static mapcache_context_apache_server* apache_server_context_create(server_rec *s, apr_pool_t *pool) -{ - mapcache_context_apache_server *ctx = apr_pcalloc(pool, sizeof(mapcache_context_apache_server)); - ctx->ctx.ctx.pool = pool; - ctx->ctx.ctx.config = NULL; - ctx->server = s; - init_apache_server_context(ctx); - return ctx; + mapcache_context_apache_request *rctx = apr_pcalloc(r->pool, sizeof(mapcache_context_apache_request)); + mapcache_context *ctx = (mapcache_context*)rctx; + mapcache_context_init(ctx); + ctx->pool = r->pool; + rctx->request = r; + ctx->log = apache_context_request_log; + ctx->clone = mapcache_context_request_clone; + return rctx; +} + +static mapcache_context_apache_server* create_apache_server_context(server_rec *s, apr_pool_t *pool) +{ + mapcache_context_apache_server *actx = apr_pcalloc(pool, sizeof(mapcache_context_apache_server)); + mapcache_context *ctx = (mapcache_context*)actx; + mapcache_context_init(ctx); + ctx->pool = pool; + ctx->config = NULL; + ctx->log = apache_context_server_log; + actx->server = s; + return actx; } /* read post body. code taken from "The apache modules book, Nick Kew" */ @@ -227,34 +218,36 @@ } } else { bytes = p->rule->max_post_len; - } + } bb = apr_brigade_create(mctx->pool, r->connection->bucket_alloc); bbin = apr_brigade_create(mctx->pool, r->connection->bucket_alloc); p->post_len = 0; do { + apr_bucket *nextb; rv = ap_get_brigade(r->input_filters, bbin, AP_MODE_READBYTES, APR_BLOCK_READ, bytes); if(rv != APR_SUCCESS) { mctx->set_error(mctx, 500, "failed to read form input"); return; } - for(b = APR_BRIGADE_FIRST(bbin); b != APR_BRIGADE_SENTINEL(bbin); b = APR_BUCKET_NEXT(b)) { + for(b = APR_BRIGADE_FIRST(bbin); b != APR_BRIGADE_SENTINEL(bbin); b = nextb) { + nextb = APR_BUCKET_NEXT(b); if(APR_BUCKET_IS_EOS(b)) { eos = 1; } - } - if(!APR_BUCKET_IS_METADATA(b)) { - if(b->length != (apr_size_t)(-1)) { - p->post_len += b->length; - if(p->post_len > p->rule->max_post_len) { - apr_bucket_delete(b); + if(!APR_BUCKET_IS_METADATA(b)) { + if(b->length != (apr_size_t)(-1)) { + p->post_len += b->length; + if(p->post_len > p->rule->max_post_len) { + apr_bucket_delete(b); + } } } - } - if(p->post_len <= p->rule->max_post_len) { - APR_BUCKET_REMOVE(b); - APR_BRIGADE_INSERT_TAIL(bb, b); + if(p->post_len <= p->rule->max_post_len) { + APR_BUCKET_REMOVE(b); + APR_BRIGADE_INSERT_TAIL(bb, b); + } } } while (!eos); @@ -264,11 +257,13 @@ } p->post_buf = apr_palloc(mctx->pool, p->post_len+1); - - rv = apr_brigade_flatten(bb, p->post_buf, &(p->post_len)); - if(rv != APR_SUCCESS) { - mctx->set_error(mctx, 500, "error (flatten) reading form data"); - return; + + if(p->post_len> 0) { + rv = apr_brigade_flatten(bb, p->post_buf, &(p->post_len)); + if(rv != APR_SUCCESS) { + mctx->set_error(mctx, 500, "error (flatten) reading form data"); + return; + } } p->post_buf[p->post_len] = 0; } @@ -312,45 +307,48 @@ static void mod_mapcache_child_init(apr_pool_t *pool, server_rec *s) { - int rv; - mapcache_server_cfg* cfg = ap_get_module_config(s->module_config, &mapcache_module); -#ifdef APR_HAS_THREADS - apr_thread_mutex_create(&thread_mutex,APR_THREAD_MUTEX_DEFAULT,pool); -#endif - rv = mapcache_connection_pool_create(&(cfg->cp),pool); - ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, "creating a mapcache connection pool for server"); - if(rv!=APR_SUCCESS) { - ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "failed to create mapcache connection pool"); + for( ; s ; s=s->next) { + mapcache_server_cfg* cfg = ap_get_module_config(s->module_config, &mapcache_module); + int i,rv; + for(i=0;ialiases->nelts;i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + rv = mapcache_connection_pool_create(&(alias_entry->cp),pool); + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, "creating a child process mapcache connection pool on server %s for alias %s", s->server_hostname, alias_entry->endpoint); + if(rv!=APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "failed to create mapcache connection pool"); + } + } + for(i=0;iquickaliases->nelts;i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->quickaliases,i,mapcache_alias_entry*); + rv = mapcache_connection_pool_create(&(alias_entry->cp),pool); + ap_log_error(APLOG_MARK, APLOG_DEBUG, 0, s, "creating a child process mapcache connection pool on server %s for alias %s", s->server_hostname, alias_entry->endpoint); + if(rv!=APR_SUCCESS) { + ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "failed to create mapcache connection pool"); + } + } } } -static int mod_mapcache_request_handler(request_rec *r) -{ +static int mapcache_handler(request_rec *r, mapcache_alias_entry *alias_entry) { apr_table_t *params; mapcache_request *request = NULL; - mapcache_context_apache_request *apache_ctx = NULL; + mapcache_context_apache_request *apache_ctx = create_apache_request_context(r); + mapcache_context *ctx = (mapcache_context*)apache_ctx; mapcache_http_response *http_response = NULL; - mapcache_context *global_ctx = NULL; - - if (!r->handler || strcmp(r->handler, "mapcache")) { - return DECLINED; - } - if (r->method_number != M_GET && r->method_number != M_POST) { - return HTTP_METHOD_NOT_ALLOWED; - } + ctx->config = alias_entry->cfg; + ctx->connection_pool = alias_entry->cp; + ctx->supports_redirects = 1; + ctx->headers_in = r->headers_in; - apache_ctx = apache_request_context_create(r); - global_ctx = (mapcache_context*)apache_ctx; - global_ctx->supports_redirects = 1; - global_ctx->headers_in = r->headers_in; + params = mapcache_http_parse_param_string(ctx, r->args); - params = mapcache_http_parse_param_string(global_ctx, r->args); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "mapcache dispatch %s",r->path_info); - mapcache_service_dispatch_request(global_ctx,&request,r->path_info,params,global_ctx->config); - if(GC_HAS_ERROR(global_ctx) || !request) { + mapcache_service_dispatch_request(ctx,&request,r->path_info,params,ctx->config); + if(GC_HAS_ERROR(ctx) || !request) { return write_http_response(apache_ctx, - mapcache_core_respond_to_error(global_ctx)); + mapcache_core_respond_to_error(ctx)); } if(request->type == MAPCACHE_REQUEST_GET_CAPABILITIES) { @@ -379,28 +377,28 @@ *end = '\0'; } } - http_response = mapcache_core_get_capabilities(global_ctx,request->service,req_caps, - url,original->path_info,global_ctx->config); + http_response = mapcache_core_get_capabilities(ctx,request->service,req_caps, + url,original->path_info,ctx->config); } else if( request->type == MAPCACHE_REQUEST_GET_TILE) { mapcache_request_get_tile *req_tile = (mapcache_request_get_tile*)request; - http_response = mapcache_core_get_tile(global_ctx,req_tile); + http_response = mapcache_core_get_tile(ctx,req_tile); } else if( request->type == MAPCACHE_REQUEST_PROXY ) { const char *buf; mapcache_request_proxy *req_proxy = (mapcache_request_proxy*)request; if(r->method_number == M_POST) { read_post_body(apache_ctx, req_proxy); - if(GC_HAS_ERROR(global_ctx)) { - return write_http_response(apache_ctx, mapcache_core_respond_to_error(global_ctx)); + if(GC_HAS_ERROR(ctx)) { + return write_http_response(apache_ctx, mapcache_core_respond_to_error(ctx)); } if(!req_proxy->headers) { - req_proxy->headers = apr_table_make(global_ctx->pool, 2); + req_proxy->headers = apr_table_make(ctx->pool, 2); } apr_table_set(req_proxy->headers, "Content-Type", r->content_type); if((buf = apr_table_get(r->headers_in,"X-Forwarded-For"))) { #if (AP_SERVER_MAJORVERSION_NUMBER == 2) && (AP_SERVER_MINORVERSION_NUMBER < 4) - apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(global_ctx->pool,"%s, %s", buf, r->connection->remote_ip)); + apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(ctx->pool,"%s, %s", buf, r->connection->remote_ip)); #else - apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(global_ctx->pool,"%s, %s", buf, r->connection->client_ip)); + apr_table_set(req_proxy->headers, "X-Forwarded-For", apr_psprintf(ctx->pool,"%s, %s", buf, r->connection->client_ip)); #endif } else { #if (AP_SERVER_MAJORVERSION_NUMBER == 2) && (AP_SERVER_MINORVERSION_NUMBER < 4) @@ -412,36 +410,98 @@ if ((buf = apr_table_get(r->headers_in, "Host"))) { const char *buf2; if((buf2 = apr_table_get(r->headers_in,"X-Forwarded-Host"))) { - apr_table_set(req_proxy->headers, "X-Forwarded-Host", apr_psprintf(global_ctx->pool,"%s, %s",buf2,buf)); + apr_table_set(req_proxy->headers, "X-Forwarded-Host", apr_psprintf(ctx->pool,"%s, %s",buf2,buf)); } else { apr_table_set(req_proxy->headers, "X-Forwarded-Host", buf); } } - + if ((buf = apr_table_get(r->headers_in, "X-Forwarded-Server"))) { - apr_table_set(req_proxy->headers, "X-Forwarded-Server", apr_psprintf(global_ctx->pool, "%s, %s", buf, r->server->server_hostname)); + apr_table_set(req_proxy->headers, "X-Forwarded-Server", apr_psprintf(ctx->pool, "%s, %s", buf, r->server->server_hostname)); } else { apr_table_set(req_proxy->headers, "X-Forwarded-Server", r->server->server_hostname); } } - http_response = mapcache_core_proxy_request(global_ctx, req_proxy); + http_response = mapcache_core_proxy_request(ctx, req_proxy); } else if( request->type == MAPCACHE_REQUEST_GET_MAP) { mapcache_request_get_map *req_map = (mapcache_request_get_map*)request; - http_response = mapcache_core_get_map(global_ctx,req_map); + http_response = mapcache_core_get_map(ctx,req_map); } else if( request->type == MAPCACHE_REQUEST_GET_FEATUREINFO) { mapcache_request_get_feature_info *req_fi = (mapcache_request_get_feature_info*)request; - http_response = mapcache_core_get_featureinfo(global_ctx,req_fi); + http_response = mapcache_core_get_featureinfo(ctx,req_fi); } else { - global_ctx->set_error(global_ctx,500,"###BUG### unknown request type"); + ctx->set_error(ctx,500,"###BUG### unknown request type"); } - if(GC_HAS_ERROR(global_ctx)) { + if(GC_HAS_ERROR(ctx)) { return write_http_response(apache_ctx, - mapcache_core_respond_to_error(global_ctx)); + mapcache_core_respond_to_error(ctx)); } return write_http_response(apache_ctx,http_response); } +static int mod_mapcache_quick_handler(request_rec *r, int lookup) { + mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias_entry *alias_entry; + int i; + + ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "mapcache quick handler hook on uri %s",r->uri); + + if (!sconfig || !sconfig->quickaliases) + return DECLINED; + + if (r->uri[0] != '/' && r->uri[0]) + return DECLINED; + + if(lookup) { + return DECLINED; + } + + /* loop through the entries to find one where the alias matches */ + for(i=0; iquickaliases->nelts; i++) { + int l; + alias_entry = APR_ARRAY_IDX(sconfig->quickaliases,i,mapcache_alias_entry*); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "cheking mapcache alias %s against %s",r->uri,alias_entry->endpoint); + + if((l=mapcache_alias_matches(r->uri, alias_entry->endpoint))>0) { + if (r->method_number != M_GET && r->method_number != M_POST) { + return HTTP_METHOD_NOT_ALLOWED; + } + r->path_info = &(r->uri[l]); + return mapcache_handler(r,alias_entry); + } + } + return DECLINED; +} + +static int mod_mapcache_request_handler(request_rec *r) +{ + const char *mapcache_alias; + int i; + mapcache_server_cfg* cfg; + if (!r->handler || strcmp(r->handler, "mapcache")) { + return DECLINED; + } + if (r->method_number != M_GET && r->method_number != M_POST) { + return HTTP_METHOD_NOT_ALLOWED; + } + cfg = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias = apr_table_get(r->notes,"mapcache_alias_entry"); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "using mapcache config %s", mapcache_config_file); + if(!mapcache_alias) { + ap_log_rerror(APLOG_MARK, APLOG_ERR, 0, r, "mapcache module bug? no mapcache_alias_entry found"); + return DECLINED; + } + + for(i=0; ialiases->nelts; i++) { + mapcache_alias_entry *alias_entry = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + if(strcmp(alias_entry->endpoint,mapcache_alias)) + continue; + return mapcache_handler(r,alias_entry); + } + return DECLINED; /*should never happen, the fixup phase would not have oriented us here*/ +} + static int mod_mapcache_post_config(apr_pool_t *p, apr_pool_t *plog, apr_pool_t *ptemp, server_rec *s) { mapcache_server_cfg* cfg = ap_get_module_config(s->module_config, &mapcache_module); @@ -449,7 +509,6 @@ ap_log_error(APLOG_MARK, APLOG_CRIT, 0, s, "configuration not found in server context"); return 1; } - #ifdef USE_VERSION_STRING ap_add_version_component(p, MAPCACHE_USERAGENT); @@ -497,35 +556,35 @@ return urip - uri; } -static int mapcache_hook_intercept(request_rec *r) +static int mapcache_hook_fixups(request_rec *r) { - mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); - apr_hash_index_t *entry; - - if (!sconfig->aliases) - return DECLINED; - - if (r->uri[0] != '/' && r->uri[0]) - return DECLINED; - - entry = apr_hash_first(r->pool,sconfig->aliases); - - /* loop through the entries to find one where the alias matches */ - while (entry) { - int l = 0; - const char *alias; - apr_ssize_t aliaslen; - mapcache_cfg *c; - apr_hash_this(entry,(const void**)&alias,&aliaslen,(void**)&c); - - if((l=mapcache_alias_matches(r->uri, c->endpoint))>0) { - r->handler = "mapcache"; - r->filename = c->configFile; - r->path_info = &(r->uri[l]); - return OK; + int i; + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "running mapcache fixup on %s (handler:%s,filename:%s)",r->uri,r->handler,r->filename); + if(!r->handler) { + mapcache_server_cfg *sconfig = ap_get_module_config(r->server->module_config, &mapcache_module); + mapcache_alias_entry *alias_entry; + + if (!sconfig || !sconfig->aliases) + return DECLINED; + + if (r->uri[0] != '/' && r->uri[0]) + return DECLINED; + + /* loop through the entries to find one where the alias matches */ + for(i=0; ialiases->nelts; i++) { + int l; + alias_entry = APR_ARRAY_IDX(sconfig->aliases,i,mapcache_alias_entry*); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "cheking mapcache alias %s against %s",r->uri,alias_entry->endpoint); + + if((l=mapcache_alias_matches(r->uri, alias_entry->endpoint))>0) { + r->handler = apr_pstrdup(r->pool,"mapcache"); + apr_table_set(r->notes,"mapcache_alias_entry",alias_entry->endpoint); + r->path_info = &(r->uri[l]); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "setting config %s for alias %s",alias_entry->configfile,alias_entry->endpoint); + //ap_log_rerror(APLOG_MARK, APLOG_DEBUG, 0, r, "using pathinfo %s from uri %s",r->path_info,r->uri); + return OK; + } } - - entry = apr_hash_next(entry); } return DECLINED; @@ -534,21 +593,19 @@ static void mod_mapcache_register_hooks(apr_pool_t *p) { - static const char * const p1[] = { "mod_alias.c", "mod_rewrite.c", NULL }; - static const char * const n1[]= { "mod_userdir.c", - "mod_vhost_alias.c", NULL - }; ap_hook_child_init(mod_mapcache_child_init, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_post_config(mod_mapcache_post_config, NULL, NULL, APR_HOOK_MIDDLE); ap_hook_handler(mod_mapcache_request_handler, NULL, NULL, APR_HOOK_MIDDLE); - ap_hook_translate_name(mapcache_hook_intercept, p1, n1, APR_HOOK_MIDDLE); + ap_hook_quick_handler(mod_mapcache_quick_handler, NULL, NULL, APR_HOOK_MIDDLE); + ap_hook_fixups(mapcache_hook_fixups, NULL, NULL, APR_HOOK_MIDDLE); } static void* mod_mapcache_create_server_conf(apr_pool_t *pool, server_rec *s) { mapcache_server_cfg *cfg = apr_pcalloc(pool, sizeof(mapcache_server_cfg)); - cfg->aliases = NULL; + cfg->aliases = apr_array_make(pool,1,sizeof(mapcache_alias_entry*)); + cfg->quickaliases = apr_array_make(pool,1,sizeof(mapcache_alias_entry*)); return cfg; } @@ -559,44 +616,84 @@ mapcache_server_cfg *vhost = (mapcache_server_cfg*)vhost_; mapcache_server_cfg *cfg = apr_pcalloc(p,sizeof(mapcache_server_cfg)); - if (base->aliases && vhost->aliases) { - cfg->aliases = apr_hash_overlay(p, vhost->aliases,base->aliases); - } else if (vhost->aliases) { - cfg->aliases = apr_hash_copy(p,vhost->aliases); - } else if (base->aliases) { - cfg->aliases = apr_hash_copy(p,base->aliases); - } - return vhost; -} - -static const char* mapcache_add_alias(cmd_parms *cmd, void *cfg, const char *alias, const char* configfile) -{ - mapcache_server_cfg *sconfig = ap_get_module_config(cmd->server->module_config, &mapcache_module); - mapcache_cfg *config = mapcache_configuration_create(cmd->pool); - mapcache_context *ctx = (mapcache_context*)apache_server_context_create(cmd->server,cmd->pool); - char *msg = NULL; - config->configFile = apr_pstrdup(cmd->pool,configfile); - config->endpoint = alias; - mapcache_configuration_parse(ctx,configfile,config,0); + cfg->aliases = apr_array_append(p, vhost->aliases,base->aliases); + cfg->quickaliases = apr_array_append(p, vhost->quickaliases,base->quickaliases); + +#if 0 + { + mapcache_alias_entry *e; + int i; + fprintf(stderr,"#### merge called ####\n"); + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(base->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge base: have alias %s on %s\n",e->configfile,e->endpoint); + } + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(vhost->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge vhosts: have alias %s on %s\n",e->configfile,e->endpoint); + } + for(i=0;ialiases->nelts;i++) { + e = APR_ARRAY_IDX(cfg->aliases,i,mapcache_alias_entry*); + fprintf(stderr,"merge result: have alias %s on %s\n",e->configfile,e->endpoint); + } + } +#endif + return cfg; +} + +static const char* mapcache_add_alias(cmd_parms *cmd, void *cfg, const char *alias, const char* configfile, const char *quick) +{ + mapcache_server_cfg *sconfig; + mapcache_alias_entry *alias_entry; + mapcache_context *ctx; + unsigned forbidden = NOT_IN_DIRECTORY|NOT_IN_FILES; + const char *err; + +#if (AP_SERVER_MAJORVERSION_NUMBER > 2) || (AP_SERVER_MINORVERSION_NUMBER >= 4) + forbidden |= NOT_IN_HTACCESS; +#endif + + err = ap_check_cmd_context(cmd, forbidden); + if (err) { + return err; + } + + sconfig = ap_get_module_config(cmd->server->module_config, &mapcache_module); + if(!sconfig || !sconfig->aliases) + return "no mapcache module config, server bug?"; + + alias_entry = apr_pcalloc(cmd->pool,sizeof(mapcache_alias_entry)); + ctx = (mapcache_context*)create_apache_server_context(cmd->server,cmd->pool); + + alias_entry->cfg = mapcache_configuration_create(cmd->pool); + alias_entry->configfile = apr_pstrdup(cmd->pool,configfile); + alias_entry->endpoint = apr_pstrdup(cmd->pool,alias); + mapcache_configuration_parse(ctx,alias_entry->configfile,alias_entry->cfg,0); if(GC_HAS_ERROR(ctx)) { return ctx->get_error_message(ctx); } - mapcache_configuration_post_config(ctx, config); + mapcache_configuration_post_config(ctx, alias_entry->cfg); if(GC_HAS_ERROR(ctx)) { return ctx->get_error_message(ctx); } - ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on alias %s", config->configFile, alias); - if(!sconfig->aliases) { - sconfig->aliases = apr_hash_make(cmd->pool); + if(mapcache_config_services_enabled(ctx, alias_entry->cfg) <= 0) { + return "no mapcache s configured/enabled, no point in continuing."; } - apr_hash_set(sconfig->aliases,configfile,APR_HASH_KEY_STRING,config); - return msg; + if(quick && !strcmp(quick,"quick")) { + APR_ARRAY_PUSH(sconfig->quickaliases,mapcache_alias_entry*) = alias_entry; + ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on (quick) endpoint %s", alias_entry->configfile, alias_entry->endpoint); + } else { + APR_ARRAY_PUSH(sconfig->aliases,mapcache_alias_entry*) = alias_entry; + ap_log_error(APLOG_MARK, APLOG_INFO, 0, cmd->server, "loaded mapcache configuration file from %s on endpoint %s", alias_entry->configfile, alias_entry->endpoint); + } + + return NULL; } static const command_rec mod_mapcache_cmds[] = { - AP_INIT_TAKE2("MapCacheAlias", mapcache_add_alias ,NULL,RSRC_CONF,"Aliased location of configuration file"), + AP_INIT_TAKE23("MapCacheAlias", mapcache_add_alias ,NULL,RSRC_CONF,"Aliased location of configuration file"), { NULL } } ; diff -Nru mapcache-1.4.0/cgi/mapcache.c mapcache-1.6.1/cgi/mapcache.c --- mapcache-1.4.0/cgi/mapcache.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/cgi/mapcache.c 2017-09-29 21:31:13.000000000 +0000 @@ -128,6 +128,11 @@ if(response->mtime) { char *datestr; char *if_modified_since = getenv("HTTP_IF_MODIFIED_SINCE"); + + datestr = apr_palloc(ctx->ctx.pool, APR_RFC822_DATE_LEN); + apr_rfc822_date(datestr, response->mtime); + printf("Last-Modified: %s\r\n", datestr); + if(if_modified_since) { apr_time_t ims_time; apr_int64_t ims,mtime; @@ -138,11 +143,14 @@ ims = apr_time_sec(ims_time); if(ims >= mtime) { printf("Status: 304 Not Modified\r\n"); + /* + * "The 304 response MUST NOT contain a message-body" + * https://tools.ietf.org/html/rfc2616#section-10.3.5 + */ + printf("\r\n"); + return; } } - datestr = apr_palloc(ctx->ctx.pool, APR_RFC822_DATE_LEN); - apr_rfc822_date(datestr, response->mtime); - printf("Last-Modified: %s\r\n", datestr); } if(response->data) { printf("Content-Length: %ld\r\n\r\n", response->data->size); @@ -192,6 +200,10 @@ if(GC_HAS_ERROR(ctx)) goto failed_load; mapcache_configuration_post_config(ctx, cfg); if(GC_HAS_ERROR(ctx)) goto failed_load; + if(mapcache_config_services_enabled(ctx,cfg) <= 0) { + ctx->set_error(ctx,500,"no mapcache s configured/enabled, no point in continuing."); + goto failed_load; + } /* no error, destroy the previous pool if we are reloading the config */ if(config_pool) { @@ -274,7 +286,6 @@ } } apr_pool_create(&(ctx->pool),config_pool); - ctx->threadlock = NULL; request = NULL; pathInfo = getenv("PATH_INFO"); diff -Nru mapcache-1.4.0/cmake/FindAPACHE.cmake mapcache-1.6.1/cmake/FindAPACHE.cmake --- mapcache-1.4.0/cmake/FindAPACHE.cmake 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/cmake/FindAPACHE.cmake 2017-09-29 21:31:13.000000000 +0000 @@ -10,7 +10,7 @@ # find_path(APACHE_INCLUDE_DIR NAMES httpd.h - PATH_SUFFIXES httpd apache apache2 + PATH_SUFFIXES httpd apache apache2 apache22 apache24 ) if(NOT DEFINED APACHE_MODULE_DIR) diff -Nru mapcache-1.4.0/CMakeLists.txt mapcache-1.6.1/CMakeLists.txt --- mapcache-1.4.0/CMakeLists.txt 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/CMakeLists.txt 2017-09-29 21:31:13.000000000 +0000 @@ -1,21 +1,19 @@ cmake_minimum_required (VERSION 2.6) -project (MapCache) +project (MapCache C) include(CheckFunctionExists) include(CheckIncludeFile) include(CheckCSourceCompiles) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall") if ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug") add_definitions(-DDEBUG) endif () set (MAPCACHE_VERSION_MAJOR 1) -set (MAPCACHE_VERSION_MINOR 4) -set (MAPCACHE_VERSION_REVISION 0) - +set (MAPCACHE_VERSION_MINOR 6) +set (MAPCACHE_VERSION_REVISION 1) if(NOT DEFINED CMAKE_INSTALL_LIBDIR) set(CMAKE_INSTALL_LIBDIR lib) @@ -58,7 +56,8 @@ check_function_exists("strncasecmp" HAVE_STRNCASECMP) check_function_exists("symlink" HAVE_SYMLINK) - +check_function_exists ("timegm" HAVE_TIMEGM) +check_function_exists ("strptime" HAVE_STRPTIME) set(CMAKE_SKIP_BUILD_RPATH FALSE) if(APPLE) @@ -67,13 +66,23 @@ set(CMAKE_LINK_INTERFACE_LIBRARY "") file(GLOB mapcache_SOURCES lib/*.c ) +file(GLOB mapcache_HEADERS include/*.h) -add_library(mapcache SHARED ${mapcache_SOURCES}) +add_library(mapcache SHARED ${mapcache_SOURCES} ${mapcache_HEADERS}) set_target_properties(mapcache PROPERTIES VERSION ${MAPCACHE_VERSION_STRING} SOVERSION 1 ) +# Add compiler flags for warnings +if(CMAKE_COMPILER_IS_GNUCC) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=declaration-after-statement") +endif() + +if(CMAKE_C_COMPILER_ID MATCHES "Clang") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=declaration-after-statement -std=c89 -Wno-comment") +endif() + #options suported by the cmake builder option(WITH_PIXMAN "Use pixman for SSE optimized image manipulations" ON) option(WITH_SQLITE "Use sqlite as a cache backend" ON) @@ -85,11 +94,12 @@ option(WITH_PCRE "Use PCRE for regex tests" OFF) option(WITH_MAPSERVER "Enable (experimental) support for the mapserver library" OFF) option(WITH_RIAK "Use Riak as a cache backend" OFF) +option(WITH_GDAL "Choose if GDAL raster support should be built in" ON) find_package(PNG) if(PNG_FOUND) include_directories(${PNG_INCLUDE_DIR}) - target_link_libraries(mapcache ${PNG_LIBRARY}) + target_link_libraries(mapcache ${PNG_LIBRARIES}) else(PNG_FOUND) report_mandatory_not_found(PNG) endif(PNG_FOUND) @@ -154,12 +164,24 @@ endif(PIXMAN_FOUND) endif (WITH_PIXMAN) +if(WITH_GDAL) + find_package(GDAL) + if(GDAL_FOUND) + include_directories(${GDAL_INCLUDE_DIR}) + target_link_libraries(mapcache ${GDAL_LIBRARY}) + set (USE_GDAL 1) + else(GDAL_FOUND) + report_optional_not_found(GDAL) + endif(GDAL_FOUND) +endif(WITH_GDAL) + if(WITH_PCRE) find_package(PCRE) if(PCRE_FOUND) include_directories(${PCRE_INCLUDE_DIR}) target_link_libraries(mapcache ${PCRE_LIBRARY}) set (USE_PCRE 1) + add_definitions(-DPCRE_STATIC) else(PCRE_FOUND) report_optional_not_found(PCRE) endif(PCRE_FOUND) @@ -291,6 +313,7 @@ status_optional_component("PCRE" "${USE_PCRE}" "${PCRE_LIBRARY}") status_optional_component("Experimental mapserver support" "${USE_MAPSERVER}" "${MAPSERVER_LIBRARY}") status_optional_component("RIAK" "${USE_RIAK}" "${RIAK_LIBRARY}") +status_optional_component("GDAL" "${USE_GDAL}" "${GDAL_LIBRARY}") INSTALL(TARGETS mapcache DESTINATION ${CMAKE_INSTALL_LIBDIR}) diff -Nru mapcache-1.4.0/debian/changelog mapcache-1.6.1/debian/changelog --- mapcache-1.4.0/debian/changelog 2016-07-03 15:22:10.000000000 +0000 +++ mapcache-1.6.1/debian/changelog 2017-11-06 02:03:11.000000000 +0000 @@ -1,8 +1,65 @@ -mapcache (1.4.0-5~trusty0) trusty; urgency=medium +mapcache (1.6.1-1~trusty0) trusty; urgency=medium - * No change rebuild for gdal 2.1.0 transition. + * No change rebuild for Trusty. - -- Angelos Tzotsos Mon, 10 May 2016 03:51:00 +0200 + -- Angelos Tzotsos Mon, 06 Nov 2017 03:00:00 +0200 + +mapcache (1.6.1-1) unstable; urgency=medium + + * New upstream release. + * Use pkg-info.mk variables instead of dpkg-parsechangelog output. + * Add autopkgtest to test installability. + * Bump Standards-Version to 4.1.1, no changes. + * Drop spelling-errors.patch, applied upstream. + * Change section for libapache2-mod-mapcache to httpd. + * Update symbols for 1.6.1. + + -- Bas Couwenberg Sat, 30 Sep 2017 09:25:05 +0200 + +mapcache (1.6.0-2) unstable; urgency=medium + + * Update symbols for other architecture. + * Move from experimental to unstable. + + -- Bas Couwenberg Sun, 18 Jun 2017 13:25:52 +0200 + +mapcache (1.6.0-1) experimental; urgency=medium + + * New upstream release. + * Fix 'contours' typo in manpage. + * Drop typo patches, applied upstream. + * Update copyright file, changes: + - Update copyright years for Regents of the University of Minnesota + - Add Frank Warmerdam to copyright holders + * Update symbols for amd64. + * Add patch to fix spelling errors. + + -- Bas Couwenberg Thu, 13 Apr 2017 18:54:23 +0200 + +mapcache (1.4.1-3) unstable; urgency=medium + + * Update symbols for hurd-i386. + * Bump Standards-Version to 3.9.8, no changes. + * Drop libmapcache2 post{inst,rm}, rely on dpkg-triggers for ldconfig. + + -- Bas Couwenberg Thu, 05 May 2016 11:40:45 +0200 + +mapcache (1.4.1-2) unstable; urgency=medium + + * Disable memcache support on hurd-i386 too. + * Update symbols for other architectures. + + -- Bas Couwenberg Fri, 26 Feb 2016 09:46:50 +0100 + +mapcache (1.4.1-1) unstable; urgency=medium + + * New upstream release. + * Update Vcs-Git URL to use HTTPS. + * Add patches for various typos. + * Bump Standards-Version to 3.9.7, no changes. + * Update symbols for 1.4.1. + + -- Bas Couwenberg Thu, 25 Feb 2016 21:04:46 +0100 mapcache (1.4.0-4) unstable; urgency=medium diff -Nru mapcache-1.4.0/debian/control mapcache-1.6.1/debian/control --- mapcache-1.4.0/debian/control 2015-08-25 13:53:59.000000000 +0000 +++ mapcache-1.6.1/debian/control 2017-09-30 07:25:05.000000000 +0000 @@ -36,9 +36,9 @@ docbook-xsl, docbook-xml, xsltproc -Standards-Version: 3.9.6 +Standards-Version: 4.1.1 Vcs-Browser: https://anonscm.debian.org/cgit/pkg-grass/mapcache.git -Vcs-Git: git://anonscm.debian.org/pkg-grass/mapcache.git +Vcs-Git: https://anonscm.debian.org/git/pkg-grass/mapcache.git Homepage: http://www.mapserver.org/mapcache/ Package: libmapcache1 @@ -104,7 +104,7 @@ Package: libapache2-mod-mapcache Architecture: any -Section: web +Section: httpd Depends: ${shlibs:Depends}, ${misc:Depends} Description: tile caching server - Apache module diff -Nru mapcache-1.4.0/debian/copyright mapcache-1.6.1/debian/copyright --- mapcache-1.4.0/debian/copyright 2015-08-25 14:09:54.000000000 +0000 +++ mapcache-1.6.1/debian/copyright 2017-04-13 18:08:01.000000000 +0000 @@ -4,7 +4,8 @@ Source: https://github.com/mapserver/mapcache/releases Files: * -Copyright: 1996-2013, Regents of the University of Minnesota +Copyright: 1996-2017, Regents of the University of Minnesota + 2004, Frank Warmerdam License: MIT Files: include/ezxml.h diff -Nru mapcache-1.4.0/debian/libmapcache1.postinst mapcache-1.6.1/debian/libmapcache1.postinst --- mapcache-1.4.0/debian/libmapcache1.postinst 2013-10-09 15:29:03.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.postinst 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -#DEBHELPER# - -if [ "$1" = configure ] ; then - ldconfig -fi - -exit 0 diff -Nru mapcache-1.4.0/debian/libmapcache1.postrm mapcache-1.6.1/debian/libmapcache1.postrm --- mapcache-1.4.0/debian/libmapcache1.postrm 2013-10-09 15:29:03.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.postrm 1970-01-01 00:00:00.000000000 +0000 @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -# Source debconf library. -. /usr/share/debconf/confmodule - -#DEBHELPER# - -if [ "$1" = remove ] ; then - ldconfig -fi - -exit 0 diff -Nru mapcache-1.4.0/debian/libmapcache1.symbols mapcache-1.6.1/debian/libmapcache1.symbols --- mapcache-1.4.0/debian/libmapcache1.symbols 2015-08-06 19:04:16.000000000 +0000 +++ mapcache-1.6.1/debian/libmapcache1.symbols 2017-09-30 07:25:05.000000000 +0000 @@ -1,4 +1,4 @@ -# SymbolsHelper-Confirmed: 1.4.0 amd64 mips powerpc s390x +# SymbolsHelper-Confirmed: 1.6.1 amd64 libmapcache.so.1 #PACKAGE# #MINVER# EZXML_NIL@Base 1.0.0 _configuration_parse_wms_xml@Base 1.0.0 @@ -20,6 +20,7 @@ _create_demo_wmts@Base 1.0.0 _error_report_wmts@Base 1.0.0 _format_error_wms@Base 1.0.0 + _mapcache_cache_rest_add_headers_from_file@Base 1.6.0 _mapcache_cache_rest_headers@Base 1.4.0 _mapcache_context_clear_error_default@Base 1.0.0 _mapcache_context_get_error_default@Base 1.0.0 @@ -30,6 +31,10 @@ _mapcache_context_set_exception_default@Base 1.0.0 _mapcache_curl_header_callback@Base 1.0.0 _mapcache_curl_memory_callback@Base 1.0.0 + _mapcache_dimension_time_get_all_entries@Base 1.6.0 + _mapcache_dimension_time_get_entries@Base 1.6.0 + _mapcache_dimension_time_get_entries_for_value@Base 1.6.0 + _mapcache_dimension_time_parse_xml@Base 1.6.0 _mapcache_imageio_classify@Base 1.0.0 _mapcache_imageio_jpeg_buffer_empty_output_buffer@Base 1.0.0 _mapcache_imageio_jpeg_buffer_term_destination@Base 1.0.0 @@ -59,21 +64,25 @@ _mapcache_source_dummy_configuration_parse_xml@Base 1.0.0 _mapcache_source_dummy_query@Base 1.0.0 _mapcache_source_dummy_render_map@Base 1.0.0 + _mapcache_source_fallback_configuration_check@Base 1.6.0 + _mapcache_source_fallback_configuration_parse_xml@Base 1.6.0 + _mapcache_source_fallback_query@Base 1.6.0 + _mapcache_source_fallback_render_map@Base 1.6.0 + _mapcache_source_gdal_configuration_check@Base 1.6.0 + _mapcache_source_gdal_configuration_parse@Base 1.6.0 + _mapcache_source_gdal_render_metatile@Base 1.6.0 _mapcache_source_wms_configuration_check@Base 1.0.0 _mapcache_source_wms_configuration_parse_xml@Base 1.0.0 _mapcache_source_wms_query@Base 1.0.0 _mapcache_source_wms_render_map@Base 1.0.0 - _mapcache_timedimension_sqlite_get_all_entries@Base 1.2.0 - _mapcache_timedimension_sqlite_get_entries@Base 1.2.0 - _mapcache_timedimension_sqlite_parse_xml@Base 1.2.0 _mapcache_unescape_url@Base 1.0.0 _wmts_inspire_metadata_responselanguages@Base 1.2.0 _wmts_service_identification_keywords@Base 1.2.0 base64_encode@Base 1.4.0 buffer_write_callback@Base 1.4.0 crc_table@Base 1.2.0 - (arch=!amd64)crc_table_computed@Base 1.4.0 - create_memcache@Base 1.4.0 + (arch=!alpha !amd64 !arm64 !armel !armhf !hurd-i386 !i386 !kfreebsd-amd64 !kfreebsd-i386 !mips64el !mipsel !ppc64el !x32)crc_table_computed@Base 1.4.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)create_memcache@Base 1.4.1 empty_png_384@Base 1.4.0 ezxml_add_child@Base 1.0.0 ezxml_ampencode@Base 1.0.0 @@ -119,19 +128,30 @@ mapcache_buffer_append@Base 1.0.0 mapcache_buffer_create@Base 1.0.0 mapcache_cache_azure_create@Base 1.4.0 + mapcache_cache_bdb_create@Base 1.6.0 mapcache_cache_composite_create@Base 1.4.0 + mapcache_cache_couchbase_create@Base 1.6.0 mapcache_cache_disk_create@Base 1.0.0 mapcache_cache_fallback_create@Base 1.4.0 mapcache_cache_google_create@Base 1.4.0 mapcache_cache_mbtiles_create@Base 1.0.0 - mapcache_cache_memcache_create@Base 1.4.0 + mapcache_cache_memcache_create@Base 1.6.0 mapcache_cache_multitier_create@Base 1.4.0 mapcache_cache_rest_create@Base 1.4.0 mapcache_cache_rest_init@Base 1.4.0 + mapcache_cache_riak_create@Base 1.6.0 mapcache_cache_s3_create@Base 1.4.0 mapcache_cache_sqlite_create@Base 1.0.0 + mapcache_cache_tc_create@Base 1.6.0 mapcache_cache_tiff_create@Base 1.0.0 + mapcache_cache_tile_delete@Base 1.6.0 + mapcache_cache_tile_exists@Base 1.6.0 + mapcache_cache_tile_get@Base 1.6.0 + mapcache_cache_tile_multi_set@Base 1.6.0 + mapcache_cache_tile_set@Base 1.6.0 mapcache_config_parse_locker@Base 1.4.0 + mapcache_config_parse_locker_old@Base 1.6.0 + mapcache_config_services_enabled@Base 1.6.0 mapcache_configuration_add_cache@Base 1.0.0 mapcache_configuration_add_grid@Base 1.0.0 mapcache_configuration_add_image_format@Base 1.0.0 @@ -158,7 +178,6 @@ mapcache_core_get_tile@Base 1.0.0 mapcache_core_proxy_request@Base 1.0.0 mapcache_core_respond_to_error@Base 1.0.0 - mapcache_dimension_intervals_create@Base 1.0.0 mapcache_dimension_regex_create@Base 1.0.0 mapcache_dimension_sqlite_create@Base 1.4.0 mapcache_dimension_time_create@Base 1.0.0 @@ -170,11 +189,12 @@ mapcache_grid_get_cell@Base 1.0.0 mapcache_grid_get_closest_wms_level@Base 1.4.0 mapcache_grid_get_crs@Base 1.0.0 - mapcache_grid_get_extent@Base 1.0.0 mapcache_grid_get_horizontal_resolution@Base 1.0.0 mapcache_grid_get_level@Base 1.0.0 + mapcache_grid_get_metatile_extent@Base 1.6.0 mapcache_grid_get_resolution@Base 1.0.0 mapcache_grid_get_srs@Base 1.0.0 + mapcache_grid_get_tile_extent@Base 1.6.0 mapcache_grid_get_vertical_resolution@Base 1.0.0 mapcache_grid_get_xy@Base 1.0.0 mapcache_http_build_url@Base 1.0.0 @@ -214,16 +234,22 @@ mapcache_locker_fallback_parse_xml@Base 1.4.0 mapcache_locker_fallback_ping_lock@Base 1.4.0 mapcache_locker_fallback_release_lock@Base 1.4.0 - mapcache_locker_memcache_aquire_lock@Base 1.4.0 - mapcache_locker_memcache_create@Base 1.4.0 - mapcache_locker_memcache_parse_xml@Base 1.4.0 - mapcache_locker_memcache_ping_lock@Base 1.4.0 - mapcache_locker_memcache_release_lock@Base 1.4.0 - mapcache_memcache_connection_constructor@Base 1.4.0 - mapcache_memcache_connection_destructor@Base 1.4.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_aquire_lock@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_create@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_parse_xml@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_ping_lock@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_locker_memcache_release_lock@Base 1.4.1 + mapcache_make_parent_dirs@Base 1.4.1 + mapcache_map_set_cached_dimension@Base 1.6.0 + mapcache_map_set_requested_dimension@Base 1.6.0 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_memcache_connection_constructor@Base 1.4.1 + (arch=!armel !armhf !hppa !hurd-i386 !i386 !mips !mipsel !powerpc)mapcache_memcache_connection_destructor@Base 1.4.1 mapcache_meters_per_unit@Base 1.0.0 mapcache_ogc_strptime@Base 1.2.0 mapcache_prefetch_tiles@Base 1.0.0 + mapcache_requested_dimensions_clone@Base 1.6.0 + mapcache_rest_connection_constructor@Base 1.6.0 + mapcache_rest_connection_destructor@Base 1.6.0 mapcache_service_demo_create@Base 1.0.0 mapcache_service_dispatch_request@Base 1.0.0 mapcache_service_gmaps_create@Base 1.0.0 @@ -233,15 +259,24 @@ mapcache_service_ve_create@Base 1.0.0 mapcache_service_wms_create@Base 1.0.0 mapcache_service_wmts_create@Base 1.0.0 + mapcache_set_cached_dimension@Base 1.6.0 + mapcache_set_requested_dimension@Base 1.6.0 mapcache_source_dummy_create@Base 1.0.0 + mapcache_source_fallback_create@Base 1.6.0 + mapcache_source_gdal_connection_constructor@Base 1.6.0 + mapcache_source_gdal_connection_destructor@Base 1.6.0 mapcache_source_gdal_create@Base 1.0.0 mapcache_source_init@Base 1.0.0 mapcache_source_mapserver_create@Base 1.0.0 + mapcache_source_query_info@Base 1.6.0 + mapcache_source_render_map@Base 1.6.0 mapcache_source_wms_create@Base 1.0.0 mapcache_sqlite_connection_constructor@Base 1.4.0 mapcache_sqlite_connection_destructor@Base 1.4.0 mapcache_sqlite_dimension_connection_constructor@Base 1.4.0 mapcache_sqlite_dimension_connection_destructor@Base 1.4.0 + mapcache_tile_set_cached_dimension@Base 1.6.0 + mapcache_tile_set_requested_dimension@Base 1.6.0 mapcache_tileset_add_watermark@Base 1.0.0 mapcache_tileset_assemble_map_tiles@Base 1.0.0 mapcache_tileset_assemble_out_of_zoom_tile@Base 1.2.0 @@ -260,16 +295,24 @@ mapcache_tileset_tile_create@Base 1.0.0 mapcache_tileset_tile_delete@Base 1.0.0 mapcache_tileset_tile_get@Base 1.0.0 + mapcache_tileset_tile_get_readonly@Base 1.6.0 + mapcache_tileset_tile_get_with_subdimensions@Base 1.6.0 + mapcache_tileset_tile_resource_key@Base 1.6.0 + mapcache_tileset_tile_set_get_with_subdimensions@Base 1.6.0 mapcache_tileset_tile_validate@Base 1.0.0 - mapcache_timedimension_get_entries_for_value@Base 1.2.0 - mapcache_timedimension_sqlite_create@Base 1.2.0 + mapcache_tileset_tile_validate_x@Base 1.6.0 + mapcache_tileset_tile_validate_y@Base 1.6.0 + mapcache_tileset_tile_validate_z@Base 1.6.0 mapcache_unlock_resource@Base 1.0.0 mapcache_util_extract_double_list@Base 1.0.0 mapcache_util_extract_int_list@Base 1.0.0 mapcache_util_get_tile_dimkey@Base 1.0.0 mapcache_util_get_tile_key@Base 1.0.0 + mapcache_util_quadkey_decode@Base 1.6.0 + mapcache_util_quadkey_encode@Base 1.6.0 mapcache_util_str_replace@Base 1.0.0 mapcache_util_str_sanitize@Base 1.0.0 + mapcache_util_str_xml_escape@Base 1.6.1 parseCache@Base 1.0.0 parseDimensions@Base 1.0.0 parseFormat@Base 1.0.0 @@ -278,7 +321,6 @@ parseServices@Base 1.0.0 parseSource@Base 1.0.0 parseTileset@Base 1.0.0 - parseTimeDimension@Base 1.2.0 relative_path@Base 1.0.0 sha256@Base 1.4.0 sha256_final@Base 1.4.0 diff -Nru mapcache-1.4.0/debian/mapcache_seed.1.xml mapcache-1.6.1/debian/mapcache_seed.1.xml --- mapcache-1.4.0/debian/mapcache_seed.1.xml 2015-07-28 17:42:59.000000000 +0000 +++ mapcache-1.6.1/debian/mapcache_seed.1.xml 2017-04-13 18:08:01.000000000 +0000 @@ -323,7 +323,7 @@ - Given a shapefile that contains the world country countours, seed only + Given a shapefile that contains the world country contours, seed only the areas that are covered by land (i.e. skip the oceans). Also use 4 request threads in parallel: diff -Nru mapcache-1.4.0/debian/patches/cmake-mapserver-include.patch mapcache-1.6.1/debian/patches/cmake-mapserver-include.patch --- mapcache-1.4.0/debian/patches/cmake-mapserver-include.patch 2015-08-25 14:17:24.000000000 +0000 +++ mapcache-1.6.1/debian/patches/cmake-mapserver-include.patch 2017-04-13 18:08:01.000000000 +0000 @@ -4,7 +4,7 @@ --- a/CMakeLists.txt +++ b/CMakeLists.txt -@@ -222,10 +222,18 @@ if(WITH_GEOTIFF) +@@ -244,10 +244,18 @@ if(WITH_GEOTIFF) endif (WITH_GEOTIFF) if(WITH_MAPSERVER) diff -Nru mapcache-1.4.0/debian/rules mapcache-1.6.1/debian/rules --- mapcache-1.4.0/debian/rules 2015-08-25 17:23:36.000000000 +0000 +++ mapcache-1.6.1/debian/rules 2017-09-27 13:14:44.000000000 +0000 @@ -20,8 +20,11 @@ CFLAGS+=$(CPPFLAGS) CFLAGS+=$(LDFLAGS) -MAPCACHE_VERSION=$(shell dpkg-parsechangelog | sed -ne 's/^Version: \(.*\)-.*/\1/p' | sed -e 's/\+.*//; s/^[0-9]://') -BUILD_DATE=$(shell dpkg-parsechangelog | sed -ne 's/^Date: //p' | LC_ALL=C date -u "+%d %B %Y" -f -) +include /usr/share/dpkg/pkg-info.mk + +UPSTREAM_VERSION = $(shell echo $(DEB_VERSION_UPSTREAM) | sed -e 's/\+.*//') + +BUILD_DATE=$(shell LC_ALL=C date -u "+%d %B %Y" -d "@$(SOURCE_DATE_EPOCH)") CMAKE_OPTS:= \ -DCMAKE_INSTALL_PREFIX=/usr \ @@ -42,7 +45,7 @@ -DWITH_OGR=1 # Disable memcache support on architectures where it's not supported -ifneq (,$(findstring $(DEB_HOST_ARCH),"armel armhf i386 mips mipsel powerpc hppa")) +ifneq (,$(findstring $(DEB_HOST_ARCH),"armel armhf i386 mips mipsel powerpc hppa hurd-i386")) CMAKE_OPTS += -DWITH_MEMCACHE=0 else CMAKE_OPTS += -DWITH_MEMCACHE=1 @@ -97,5 +100,5 @@ dh_install --autodest --list-missing override_dh_makeshlibs: - dh_makeshlibs -- -c0 -v$(MAPCACHE_VERSION) + dh_makeshlibs -- -c0 -v$(UPSTREAM_VERSION) diff -Nru mapcache-1.4.0/debian/tests/control mapcache-1.6.1/debian/tests/control --- mapcache-1.4.0/debian/tests/control 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/debian/tests/control 2017-09-27 13:15:10.000000000 +0000 @@ -0,0 +1,3 @@ +# Test installability +Depends: @ +Test-Command: /bin/true diff -Nru mapcache-1.4.0/include/mapcache-config.h.in mapcache-1.6.1/include/mapcache-config.h.in --- mapcache-1.4.0/include/mapcache-config.h.in 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/include/mapcache-config.h.in 2017-09-29 21:31:13.000000000 +0000 @@ -12,9 +12,11 @@ #cmakedefine USE_PCRE 1 #cmakedefine USE_MAPSERVER 1 #cmakedefine USE_RIAK 1 +#cmakedefine USE_GDAL 1 #cmakedefine HAVE_STRNCASECMP 1 #cmakedefine HAVE_SYMLINK 1 - +#cmakedefine HAVE_STRPTIME 1 +#cmakedefine HAVE_TIMEGM 1 #endif diff -Nru mapcache-1.4.0/include/mapcache.h mapcache-1.6.1/include/mapcache.h --- mapcache-1.4.0/include/mapcache.h 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/include/mapcache.h 2017-09-29 21:31:13.000000000 +0000 @@ -1,7 +1,6 @@ /****************************************************************************** - * $Id$ * - * Project: MapServer + * Project: MapCache * Author: Thomas Bonfort and the MapServer team. * ****************************************************************************** @@ -46,34 +45,11 @@ #include "errors.h" -#if 0 -#ifdef USE_GDAL -#include -#include -#endif -#endif #include #include #include -#ifdef USE_PCRE -#include -#else -#include -#endif - -#ifdef USE_MEMCACHE -#include -#endif - -#ifdef USE_COUCHBASE -#include -#endif - -#ifdef USE_RIAK -#include -#endif #define MAPCACHE_SUCCESS 0 #define MAPCACHE_FAILURE 1 @@ -90,7 +66,11 @@ #define MAPCACHE_USERAGENT "mod-mapcache/"MAPCACHE_VERSION - +#if defined(_WIN32) && !defined(__CYGWIN__) +# define MS_DLL_EXPORT __declspec(dllexport) +#else +#define MS_DLL_EXPORT +#endif typedef struct mapcache_image_format mapcache_image_format; @@ -109,44 +89,16 @@ typedef struct mapcache_request_get_feature_info mapcache_request_get_feature_info; typedef struct mapcache_map mapcache_map; typedef struct mapcache_http_response mapcache_http_response; -typedef struct mapcache_source_wms mapcache_source_wms; -#if 0 -typedef struct mapcache_source_gdal mapcache_source_gdal; -#endif -typedef struct mapcache_cache_disk mapcache_cache_disk; -typedef struct mapcache_cache_composite mapcache_cache_composite; -typedef struct mapcache_cache_fallback mapcache_cache_fallback; -typedef struct mapcache_cache_multitier mapcache_cache_multitier; -typedef struct mapcache_cache_rest mapcache_cache_rest; -typedef struct mapcache_cache_s3 mapcache_cache_s3; -typedef struct mapcache_cache_azure mapcache_cache_azure; -typedef struct mapcache_cache_google mapcache_cache_google; -#ifdef USE_TIFF -typedef struct mapcache_cache_tiff mapcache_cache_tiff; -#endif typedef struct mapcache_http mapcache_http; typedef struct mapcache_request mapcache_request; typedef struct mapcache_request_image mapcache_request_image; typedef struct mapcache_request_proxy mapcache_request_proxy; typedef struct mapcache_request_get_capabilities mapcache_request_get_capabilities; -typedef struct mapcache_request_get_capabilities_demo mapcache_request_get_capabilities_demo; -typedef struct mapcache_request_get_capabilities_wms mapcache_request_get_capabilities_wms; -typedef struct mapcache_request_get_capabilities_wmts mapcache_request_get_capabilities_wmts; typedef struct mapcache_forwarding_rule mapcache_forwarding_rule; -typedef struct mapcache_request_get_capabilities_tms mapcache_request_get_capabilities_tms; -typedef struct mapcache_request_get_capabilities_kml mapcache_request_get_capabilities_kml; typedef struct mapcache_request_get_tile mapcache_request_get_tile; typedef struct mapcache_request_get_map mapcache_request_get_map; typedef struct mapcache_service mapcache_service; -typedef struct mapcache_service_wms mapcache_service_wms; -typedef struct mapcache_service_wmts mapcache_service_wmts; -typedef struct mapcache_service_gmaps mapcache_service_gmaps; -typedef struct mapcache_service_ve mapcache_service_ve; -typedef struct mapcache_service_tms mapcache_service_tms; -typedef struct mapcache_service_kml mapcache_service_kml; -typedef struct mapcache_service_mapguide mapcache_service_mapguide; -typedef struct mapcache_service_demo mapcache_service_demo; typedef struct mapcache_server_cfg mapcache_server_cfg; typedef struct mapcache_image mapcache_image; typedef struct mapcache_grid mapcache_grid; @@ -154,17 +106,28 @@ typedef struct mapcache_grid_link mapcache_grid_link; typedef struct mapcache_context mapcache_context; typedef struct mapcache_dimension mapcache_dimension; -typedef struct mapcache_dimension_time mapcache_dimension_time; -typedef struct mapcache_timedimension mapcache_timedimension; -typedef struct mapcache_dimension_intervals mapcache_dimension_intervals; -typedef struct mapcache_dimension_values mapcache_dimension_values; -typedef struct mapcache_dimension_sqlite mapcache_dimension_sqlite; -typedef struct mapcache_dimension_regex mapcache_dimension_regex; +typedef struct mapcache_requested_dimension mapcache_requested_dimension; typedef struct mapcache_extent mapcache_extent; typedef struct mapcache_extent_i mapcache_extent_i; typedef struct mapcache_connection_pool mapcache_connection_pool; typedef struct mapcache_locker mapcache_locker; +typedef enum { + MAPCACHE_DIMENSION_VALUES, + MAPCACHE_DIMENSION_REGEX, + MAPCACHE_DIMENSION_INTERVALS, + MAPCACHE_DIMENSION_TIME, + MAPCACHE_DIMENSION_SQLITE +} mapcache_dimension_type; + +typedef enum { + MAPCACHE_DIMENSION_ASSEMBLY_NONE, + MAPCACHE_DIMENSION_ASSEMBLY_STACK, + MAPCACHE_DIMENSION_ASSEMBLY_ANIMATE +} mapcache_dimension_assembly_type; + + + /** \defgroup utility Utility */ /** @{ */ @@ -203,7 +166,7 @@ void (*set_exception)(mapcache_context *ctx, char *key, char *message, ...); /** - * \brief query context to know if an error has occured + * \brief query context to know if an error has occurred * \memberof mapcache_context */ int (*get_error)(mapcache_context * ctx); @@ -219,13 +182,13 @@ * \memberof mapcache_context */ void (*clear_errors)(mapcache_context * ctx); - + /** * \brief clear current error and store it in mapcache_error * \memberof mapcache_context */ void (*pop_errors)(mapcache_context * ctx, void **error); - + /** * \brief restore error status from mapcache_error * \memberof mapcache_context @@ -243,7 +206,6 @@ mapcache_context* (*clone)(mapcache_context *ctx); apr_pool_t *pool; mapcache_connection_pool *connection_pool; - void *threadlock; char *_contenttype; char *_errmsg; int _errcode; @@ -254,8 +216,8 @@ apr_table_t *headers_in; }; -void mapcache_context_init(mapcache_context *ctx); -void mapcache_context_copy(mapcache_context *src, mapcache_context *dst); +MS_DLL_EXPORT void mapcache_context_init(mapcache_context *ctx); +MS_DLL_EXPORT void mapcache_context_copy(mapcache_context *src, mapcache_context *dst); #define GC_CHECK_ERROR_RETURN(ctx) if(((mapcache_context*)ctx)->_errcode) return MAPCACHE_FAILURE; #define GC_CHECK_ERROR(ctx) if(((mapcache_context*)ctx)->_errcode) return; @@ -303,7 +265,8 @@ MAPCACHE_SOURCE_WMS, MAPCACHE_SOURCE_MAPSERVER, MAPCACHE_SOURCE_DUMMY, - MAPCACHE_SOURCE_GDAL + MAPCACHE_SOURCE_GDAL, + MAPCACHE_SOURCE_FALLBACK } mapcache_source_type; /**\interface mapcache_source @@ -314,6 +277,8 @@ mapcache_extent data_extent; /**< extent in which this source can produce data */ mapcache_source_type type; apr_table_t *metadata; + unsigned int retry_count; + double retry_delay; apr_array_header_t *info_formats; /** @@ -321,11 +286,11 @@ * * sets the mapcache_metatile::tile::data for the given tile */ - void (*render_map)(mapcache_context *ctx, mapcache_map *map); + void (*_render_map)(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map); - void (*query_info)(mapcache_context *ctx, mapcache_feature_info *fi); + void (*_query_info)(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi); - void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_source * source); + void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_source * source, mapcache_cfg *config); void (*configuration_check)(mapcache_context *ctx, mapcache_cfg *cfg, mapcache_source * source); }; @@ -342,81 +307,23 @@ /* TODO: authentication */ }; -/**\class mapcache_source_wms - * \brief WMS mapcache_source - * \implements mapcache_source - */ -struct mapcache_source_wms { - mapcache_source source; - apr_table_t *wms_default_params; /**< default WMS parameters (SERVICE,REQUEST,STYLES,VERSION) */ - apr_table_t *getmap_params; /**< WMS parameters specified in configuration */ - apr_table_t *getfeatureinfo_params; /**< WMS parameters specified in configuration */ - mapcache_http *http; -}; -#ifdef USE_MAPSERVER -/**\class mapcache_source_mapserver - * \brief WMS mapcache_source - * \implements mapcache_source - */ -typedef struct mapcache_source_mapserver mapcache_source_mapserver; -struct mapcache_source_mapserver { - mapcache_source source; - char *mapfile; -}; -#endif - -typedef struct mapcache_source_dummy mapcache_source_dummy; -struct mapcache_source_dummy { - mapcache_source source; - char *mapfile; - void *mapobj; -}; -#if 0 -#ifdef USE_GDAL -/**\class mapcache_source_gdal - * \brief GDAL mapcache_source - * \implements mapcache_source - */ -struct mapcache_source_gdal { - mapcache_source source; - char *datastr; /**< the gdal source string*/ - apr_table_t *gdal_params; /**< GDAL parameters specified in configuration */ - GDALDatasetH *poDataset; -}; -#endif -/** @} */ -#endif /** \defgroup cache Caches */ /** @{ */ typedef enum { - MAPCACHE_CACHE_DISK, - MAPCACHE_CACHE_REST -#ifdef USE_MEMCACHE + MAPCACHE_CACHE_DISK + ,MAPCACHE_CACHE_REST ,MAPCACHE_CACHE_MEMCACHE -#endif -#ifdef USE_SQLITE ,MAPCACHE_CACHE_SQLITE -#endif -#ifdef USE_BDB ,MAPCACHE_CACHE_BDB -#endif -#ifdef USE_TC ,MAPCACHE_CACHE_TC -#endif -#ifdef USE_TIFF ,MAPCACHE_CACHE_TIFF -#endif ,MAPCACHE_CACHE_COMPOSITE -#ifdef USE_COUCHBASE ,MAPCACHE_CACHE_COUCHBASE -#endif -#ifdef USE_RIAK ,MAPCACHE_CACHE_RIAK -#endif } mapcache_cache_type; /** \interface mapcache_cache @@ -426,293 +333,77 @@ char *name; /**< key this cache is referenced by */ mapcache_cache_type type; apr_table_t *metadata; + unsigned int retry_count; + double retry_delay; + /** * get tile content from cache - * \returns MAPCACHE_SUCCESS if the data was correctly loaded from the disk + * \returns MAPCACHE_SUCCESS if the data was correctly loaded from the cache * \returns MAPCACHE_FAILURE if the file exists but contains no data - * \returns MAPCACHE_CACHE_MISS if the file does not exist on the disk + * \returns MAPCACHE_CACHE_MISS if the file does not exist on the cache * \memberof mapcache_cache */ - int (*tile_get)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); + int (*_tile_get)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); /** * delete tile from cache * * \memberof mapcache_cache */ - void (*tile_delete)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); + void (*_tile_delete)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); - int (*tile_exists)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); + int (*_tile_exists)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); /** * set tile content to cache * \memberof mapcache_cache */ - void (*tile_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); - void (*tile_multi_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles); + void (*_tile_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile * tile); + void (*_tile_multi_set)(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles); void (*configuration_parse_xml)(mapcache_context *ctx, ezxml_t xml, mapcache_cache * cache, mapcache_cfg *config); void (*configuration_post_config)(mapcache_context *ctx, mapcache_cache * cache, mapcache_cfg *config); }; -/**\class mapcache_cache_disk - * \brief a mapcache_cache on a filesytem - * \implements mapcache_cache - */ -struct mapcache_cache_disk { - mapcache_cache cache; - char *base_directory; - char *filename_template; - int symlink_blank; - int creation_retry; - - /** - * Set filename for a given tile - * \memberof mapcache_cache_disk - */ - void (*tile_key)(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path); -}; - -typedef struct mapcache_cache_composite_cache_link mapcache_cache_composite_cache_link; -struct mapcache_cache_composite_cache_link { - mapcache_cache *cache; - int minzoom; - int maxzoom; - apr_array_header_t *grids; - apr_array_header_t *dimensions; //TODO -}; - -struct mapcache_cache_composite { - mapcache_cache cache; - apr_array_header_t *cache_links; -}; - -struct mapcache_cache_fallback { - mapcache_cache cache; - apr_array_header_t *caches; -}; +MS_DLL_EXPORT int mapcache_cache_tile_get(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +void mapcache_cache_tile_delete(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +MS_DLL_EXPORT int mapcache_cache_tile_exists(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_cache_tile_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile); +void mapcache_cache_tile_multi_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles); -struct mapcache_cache_multitier { - mapcache_cache cache; - apr_array_header_t *caches; -}; - - -typedef enum { - MAPCACHE_REST_METHOD_GET, - MAPCACHE_REST_METHOD_HEAD, - MAPCACHE_REST_METHOD_PUT, - MAPCACHE_REST_METHOD_POST, - MAPCACHE_REST_METHOD_DELETE -} mapcache_rest_method; - -typedef enum { - MAPCACHE_REST_PROVIDER_NONE, - MAPCACHE_REST_PROVIDER_S3, - MAPCACHE_REST_PROVIDER_AZURE, - MAPCACHE_REST_PROVIDER_GOOGLE, -} mapcache_rest_provider; -void sha256(const unsigned char *message, unsigned int len, unsigned char *digest); -void hmac_sha256(const unsigned char *message, unsigned int message_len, - const unsigned char *key, unsigned int key_size, - unsigned char *mac, unsigned mac_size); -void hmac_sha1(const char *message, unsigned int message_len, - const unsigned char *key, unsigned int key_size, - void *mac); -void sha_hex_encode(unsigned char *sha, unsigned int sha_size); -char *base64_encode(apr_pool_t *pool, const unsigned char *data, size_t input_length); - -typedef struct mapcache_rest_operation mapcache_rest_operation; -struct mapcache_rest_operation { - apr_table_t *headers; - mapcache_rest_method method; - char *tile_url; - void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); -}; - -typedef struct mapcache_rest_configuration mapcache_rest_configuration; -struct mapcache_rest_configuration { - apr_table_t *common_headers; - char *tile_url; - mapcache_rest_operation has_tile; - mapcache_rest_operation get_tile; - mapcache_rest_operation set_tile; - mapcache_rest_operation multi_set_tile; - mapcache_rest_operation delete_tile; - void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); -}; - -/**\class mapcache_cache_rest - * \brief a mapcache_cache on a 3rd party HTTP Rest API - * \implements mapcache_cache - */ -struct mapcache_cache_rest { - mapcache_cache cache; - mapcache_rest_configuration rest; - int use_redirects; - int retry_count; - mapcache_rest_provider provider; -}; - -struct mapcache_cache_s3 { - mapcache_cache_rest cache; - char *id; - char *secret; - char *region; -}; - -struct mapcache_cache_azure { - mapcache_cache_rest cache; - char *id; - char *secret; - char *container; -}; - -struct mapcache_cache_google { - mapcache_cache_rest cache; - char *access; - char *secret; -}; - -#ifdef USE_TIFF -struct mapcache_cache_tiff { - mapcache_cache cache; - char *filename_template; - char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; - int count_x; - int count_y; - mapcache_image_format_jpeg *format; - mapcache_locker *locker; -}; -#endif - -#ifdef USE_SQLITE -/**\class mapcache_cache_sqlite - * \brief a mapcache_cache on a filesytem - * \implements mapcache_cache - */ -typedef struct mapcache_cache_sqlite mapcache_cache_sqlite; -typedef struct mapcache_cache_sqlite_stmt mapcache_cache_sqlite_stmt; - -struct mapcache_cache_sqlite_stmt { - char *sql; -}; - -struct sqlite_conn; - -struct mapcache_cache_sqlite { - mapcache_cache cache; - char *dbfile; - mapcache_cache_sqlite_stmt create_stmt; - mapcache_cache_sqlite_stmt exists_stmt; - mapcache_cache_sqlite_stmt get_stmt; - mapcache_cache_sqlite_stmt set_stmt; - mapcache_cache_sqlite_stmt delete_stmt; - apr_table_t *pragmas; - void (*bind_stmt)(mapcache_context *ctx, void *stmt, mapcache_cache_sqlite *cache, mapcache_tile *tile); - int n_prepared_statements; - int detect_blank; - char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; - int count_x, count_y; -}; /** * \memberof mapcache_cache_sqlite */ mapcache_cache* mapcache_cache_sqlite_create(mapcache_context *ctx); mapcache_cache* mapcache_cache_mbtiles_create(mapcache_context *ctx); -#endif -#ifdef USE_BDB -typedef struct mapcache_cache_bdb mapcache_cache_bdb; -struct mapcache_cache_bdb { - mapcache_cache cache; - char *basedir; - char *key_template; -}; +/** + * \memberof mapcache_cache_bdb + */ mapcache_cache *mapcache_cache_bdb_create(mapcache_context *ctx); -#endif - -#ifdef USE_TC -typedef struct mapcache_cache_tc mapcache_cache_tc; -struct mapcache_cache_tc { - mapcache_cache cache; - char *basedir; - char *key_template; - mapcache_context *ctx; -}; -mapcache_cache *mapcache_cache_tc_create(mapcache_context *ctx); -#endif - -#ifdef USE_MEMCACHE -typedef struct mapcache_cache_memcache mapcache_cache_memcache; -/**\class mapcache_cache_memcache - * \brief a mapcache_cache on memcached servers - * \implements mapcache_cache - */ - -struct mapcache_cache_memcache_server { - char* host; - int port; -}; - -struct mapcache_cache_memcache { - mapcache_cache cache; - int nservers; - struct mapcache_cache_memcache_server *servers; - int detect_blank; -}; /** * \memberof mapcache_cache_memcache */ mapcache_cache* mapcache_cache_memcache_create(mapcache_context *ctx); -#endif - -#ifdef USE_COUCHBASE -typedef struct mapcache_cache_couchbase mapcache_cache_couchbase; - -/**\class mapcache_cache_couchbase - * \brief a mapcache_cache on couchbase servers - * \implements mapcache_cache - */ -struct mapcache_cache_couchbase { - mapcache_cache cache; -// apr_reslist_t *connection_pool; - char *host; - char *username; - char *password; - char *bucket; - mapcache_context *ctx; -}; /** * \memberof mapcache_cache_couchbase */ mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx); -#endif -#ifdef USE_RIAK -typedef struct mapcache_cache_riak mapcache_cache_riak; - -/**\class mapcache_cache_riak - * \brief a mapcache_cache for riak servers - * \implements mapcache_cache - */ -struct mapcache_cache_riak { - mapcache_cache cache; - char *host; - int port; - RIACK_STRING bucket; -}; +/** + * \memberof mapcache_cache_tc + */ +mapcache_cache* mapcache_cache_tc_create(mapcache_context *ctx); /** * \memberof mapcache_cache_riak */ mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx); -#endif /** @} */ @@ -779,7 +470,7 @@ struct mapcache_map { mapcache_tileset *tileset; mapcache_grid_link *grid_link; - apr_table_t *dimensions; + apr_array_header_t *dimensions; mapcache_buffer *encoded_data; mapcache_image *raw_image; int nodata; /**< \sa mapcache_tile::nodata */ @@ -823,36 +514,7 @@ char *mime_type; }; -struct mapcache_request_get_capabilities_tms { - mapcache_request_get_capabilities request; - mapcache_tileset *tileset; - mapcache_grid_link *grid_link; - char *version; -}; - -struct mapcache_request_get_capabilities_kml { - mapcache_request_get_capabilities request; - mapcache_tile *tile; - mapcache_tileset *tileset; - mapcache_grid_link *grid; -}; - -struct mapcache_request_get_capabilities_wms { - mapcache_request_get_capabilities request; -}; -struct mapcache_request_get_capabilities_wmts { - mapcache_request_get_capabilities request; -}; - -/** - * the capabilities request for a specific service, to be able to create - * demo pages specific to a given service - */ -struct mapcache_request_get_capabilities_demo { - mapcache_request_get_capabilities request; - mapcache_service *service; -}; struct mapcache_forwarding_rule { char *name; @@ -928,67 +590,9 @@ char **err_body, apr_table_t *headers); }; -/**\class mapcache_service_wms - * \brief an OGC WMS service - * \implements mapcache_service - */ -struct mapcache_service_wms { - mapcache_service service; - int maxsize; - apr_array_header_t *forwarding_rules; - mapcache_getmap_strategy getmap_strategy; - mapcache_resample_mode resample_mode; - mapcache_image_format *getmap_format; - int allow_format_override; /* can the client specify which image format should be returned */ -}; -/**\class mapcache_service_kml - * \brief a KML superoverlay service - * \implements mapcache_service - */ -struct mapcache_service_kml { - mapcache_service service; -}; -/**\class mapcache_service_tms - * \brief a TMS service - * \implements mapcache_service - */ -struct mapcache_service_tms { - mapcache_service service; - int reverse_y; -}; -struct mapcache_service_mapguide { - mapcache_service service; - int rows_per_folder; - int cols_per_folder; -}; - -/**\class mapcache_service_wmts - * \brief a WMTS service - * \implements mapcache_service - */ -struct mapcache_service_wmts { - mapcache_service service; -}; - -/**\class mapcache_service_demo - * \brief a demo service - * \implements mapcache_service - */ -struct mapcache_service_demo { - mapcache_service service; - -}; - -/**\class mapcache_service_ve - * \brief a virtualearth service - * \implements mapcache_service - */ -struct mapcache_service_ve { - mapcache_service service; -}; /** * \brief create and initialize a mapcache_service_wms @@ -1041,11 +645,15 @@ /** * \brief return the request that corresponds to the given url */ -void mapcache_service_dispatch_request(mapcache_context *ctx, +MS_DLL_EXPORT void mapcache_service_dispatch_request(mapcache_context *ctx, mapcache_request **request, char *pathinfo, apr_table_t *params, mapcache_cfg *config); +/** + * \brief return the number of enabled/configured services + */ +MS_DLL_EXPORT int mapcache_config_services_enabled(mapcache_context *ctx, mapcache_cfg *config); /** @} */ @@ -1136,7 +744,7 @@ /** * \brief check if image has some non opaque pixels */ -int mapcache_image_has_alpha(mapcache_image *img); +int mapcache_image_has_alpha(mapcache_image *img, unsigned int cutoff); void mapcache_image_fill(mapcache_context *ctx, mapcache_image *image, const unsigned char *fill_color); @@ -1147,17 +755,13 @@ /** @{ */ void mapcache_http_do_request(mapcache_context *ctx, mapcache_http *req, mapcache_buffer *data, apr_table_t *headers, long *http_code); char* mapcache_http_build_url(mapcache_context *ctx, char *base, apr_table_t *params); -apr_table_t *mapcache_http_parse_param_string(mapcache_context *ctx, char *args); +MS_DLL_EXPORT apr_table_t *mapcache_http_parse_param_string(mapcache_context *ctx, char *args); /** @} */ /** \defgroup configuration Configuration*/ /** @{ */ -struct mapcache_server_cfg { - apr_hash_t *aliases; /**< list of mapcache configurations aliased to a server uri */ - mapcache_connection_pool *cp; -}; @@ -1182,59 +786,28 @@ struct mapcache_locker{ mapcache_lock_result (*aquire_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock); - mapcache_lock_result (*ping_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock); - void (*release_lock)(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock); - + mapcache_lock_result (*ping_lock)(mapcache_context *ctx, mapcache_locker *self, void *lock); + void (*release_lock)(mapcache_context *ctx, mapcache_locker *self, void *lock); + void (*parse_xml)(mapcache_context *ctx, mapcache_locker *self, ezxml_t node); mapcache_lock_mode type; double timeout; double retry_interval; /* time to wait before checking again on a lock, in seconds */ }; -typedef struct { - mapcache_locker locker; - - /** - * directory where lock files will be placed. - * Must be readable and writable by the apache user. - * Must be placed on a network mounted shared directory if multiple mapcache instances - * need to be synchronized - */ - const char *dir; -} mapcache_locker_disk; - mapcache_locker* mapcache_locker_disk_create(mapcache_context *ctx); -#ifdef USE_MEMCACHE -typedef struct { - char *host; - int port; -} mapcache_locker_memcache_server; - -typedef struct { - mapcache_locker locker; - int nservers; - mapcache_locker_memcache_server *servers; -} mapcache_locker_memcache; - mapcache_locker* mapcache_locker_memcache_create(mapcache_context *ctx); -#endif - -typedef struct { - mapcache_locker locker; - apr_array_header_t *lockers; -} mapcache_locker_fallback; mapcache_locker* mapcache_locker_fallback_create(mapcache_context *ctx); void mapcache_config_parse_locker(mapcache_context *ctx, ezxml_t node, mapcache_locker **locker); +void mapcache_config_parse_locker_old(mapcache_context *ctx, ezxml_t doc, mapcache_cfg *config); /** * a configuration that will be served */ struct mapcache_cfg { - char *configFile; /**< the filename from which this configuration was loaded */ - /** * a list of services that will be responded to */ @@ -1290,11 +863,6 @@ int threaded_fetching; - /** - * the uri where the base of the service is mapped - */ - const char *endpoint; - /* for fastcgi only */ int autoreload; /* should the modification time of the config file be recorded and the file be reparsed if it is modified. */ @@ -1315,14 +883,14 @@ * @param pool * @return */ -void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi); -void mapcache_configuration_post_config(mapcache_context *ctx, mapcache_cfg *config); +MS_DLL_EXPORT void mapcache_configuration_parse(mapcache_context *ctx, const char *filename, mapcache_cfg *config, int cgi); +MS_DLL_EXPORT void mapcache_configuration_post_config(mapcache_context *ctx, mapcache_cfg *config); void mapcache_configuration_parse_xml(mapcache_context *ctx, const char *filename, mapcache_cfg *config); -mapcache_cfg* mapcache_configuration_create(apr_pool_t *pool); +MS_DLL_EXPORT mapcache_cfg* mapcache_configuration_create(apr_pool_t *pool); mapcache_source* mapcache_configuration_get_source(mapcache_cfg *config, const char *key); -mapcache_cache* mapcache_configuration_get_cache(mapcache_cfg *config, const char *key); +MS_DLL_EXPORT mapcache_cache* mapcache_configuration_get_cache(mapcache_cfg *config, const char *key); mapcache_grid *mapcache_configuration_get_grid(mapcache_cfg *config, const char *key); -mapcache_tileset* mapcache_configuration_get_tileset(mapcache_cfg *config, const char *key); +MS_DLL_EXPORT mapcache_tileset* mapcache_configuration_get_tileset(mapcache_cfg *config, const char *key); mapcache_image_format *mapcache_configuration_get_image_format(mapcache_cfg *config, const char *key); void mapcache_configuration_add_image_format(mapcache_cfg *config, mapcache_image_format *format, const char * key); void mapcache_configuration_add_source(mapcache_cfg *config, mapcache_source *source, const char * key); @@ -1337,21 +905,31 @@ void mapcache_source_init(mapcache_context *ctx, mapcache_source *source); /** + * \memberof mapcache_source + */ +void mapcache_source_render_map(mapcache_context *ctx, mapcache_source *source, mapcache_map *map); +void mapcache_source_query_info(mapcache_context *ctx, mapcache_source *source, + mapcache_feature_info *fi); + +/** * \memberof mapcache_source_gdal */ mapcache_source* mapcache_source_gdal_create(mapcache_context *ctx); /** + * \memberof mapcache_source_fallback + */ +mapcache_source* mapcache_source_fallback_create(mapcache_context *ctx); + +/** * \memberof mapcache_source_wms */ mapcache_source* mapcache_source_wms_create(mapcache_context *ctx); -#ifdef USE_MAPSERVER /** * \memberof mapcache_source_wms */ mapcache_source* mapcache_source_mapserver_create(mapcache_context *ctx); -#endif mapcache_source* mapcache_source_dummy_create(mapcache_context *ctx); @@ -1368,12 +946,10 @@ mapcache_cache* mapcache_cache_azure_create(mapcache_context *ctx); mapcache_cache* mapcache_cache_google_create(mapcache_context *ctx); -#ifdef USE_TIFF /** * \memberof mapcache_cache_tiff */ mapcache_cache* mapcache_cache_tiff_create(mapcache_context *ctx); -#endif mapcache_cache* mapcache_cache_composite_create(mapcache_context *ctx); mapcache_cache* mapcache_cache_fallback_create(mapcache_context *ctx); @@ -1407,7 +983,8 @@ apr_time_t mtime; /**< last modification time */ int expires; /**< time in seconds after which the tile should be rechecked for validity */ - apr_table_t *dimensions; + apr_array_header_t *dimensions; + /** * flag stating the tile is empty (i.e. fully transparent). * if set, this indicates that there was no error per se, but that there was @@ -1446,7 +1023,7 @@ MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT, MAPCACHE_GRID_ORIGIN_TOP_LEFT, MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT, - MAPCACHE_GRID_ORIGIN_TOP_RIGHT, + MAPCACHE_GRID_ORIGIN_TOP_RIGHT } mapcache_grid_origin; struct mapcache_grid { @@ -1478,7 +1055,7 @@ mapcache_extent *restricted_extent; mapcache_extent_i *grid_limits; int minz,maxz; - + /** * tiles above this zoom level will not be stored to the cache, but will be * dynamically generated (either by reconstructing from lower level tiles, or @@ -1538,6 +1115,7 @@ int auto_expire; int read_only; + int subdimension_read_only; /** * the cache in which the tiles should be stored @@ -1559,7 +1137,9 @@ */ apr_array_header_t *dimensions; - mapcache_timedimension *timedimension; + int store_dimension_assemblies;/**< should multiple sub-dimensions be assembled dynamically (per-request) or should they be cached once assembled */ + + mapcache_dimension_assembly_type dimension_assembly_type; /** * image to be used as a watermark @@ -1606,6 +1186,9 @@ * @return */ void mapcache_tileset_tile_validate(mapcache_context *ctx, mapcache_tile *tile); +void mapcache_tileset_tile_validate_z(mapcache_context *ctx, mapcache_tile *tile); +void mapcache_tileset_tile_validate_x(mapcache_context *ctx, mapcache_tile *tile); +void mapcache_tileset_tile_validate_y(mapcache_context *ctx, mapcache_tile *tile); /** * compute level for a given resolution @@ -1620,13 +1203,14 @@ void mapcache_tileset_get_level(mapcache_context *ctx, mapcache_tileset *tileset, double *resolution, int *level); mapcache_grid_link* mapcache_grid_get_closest_wms_level(mapcache_context *ctx, mapcache_grid_link *grid, double resolution, int *level); -void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile); /** * \brief delete tile from cache * @param whole_metatile delete all the other tiles from the metatile to */ -void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile); +MS_DLL_EXPORT void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile); int mapcache_grid_is_bbox_aligned(mapcache_context *ctx, mapcache_grid *grid, mapcache_extent *bbox); @@ -1637,7 +1221,7 @@ * @param pool * @return */ -mapcache_tile* mapcache_tileset_tile_create(apr_pool_t *pool, mapcache_tileset *tileset, mapcache_grid_link *grid_link); +MS_DLL_EXPORT mapcache_tile* mapcache_tileset_tile_create(apr_pool_t *pool, mapcache_tileset *tileset, mapcache_grid_link *grid_link); mapcache_tile* mapcache_tileset_tile_clone(apr_pool_t *pool, mapcache_tile *src); @@ -1670,27 +1254,27 @@ void mapcache_tileset_add_watermark(mapcache_context *ctx, mapcache_tileset *tileset, const char *filename); -int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void **lock); -void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void *lock); +MS_DLL_EXPORT int mapcache_lock_or_wait_for_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void **lock); +MS_DLL_EXPORT void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, void *lock); -mapcache_metatile* mapcache_tileset_metatile_get(mapcache_context *ctx, mapcache_tile *tile); -void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt); -char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt); +MS_DLL_EXPORT mapcache_metatile* mapcache_tileset_metatile_get(mapcache_context *ctx, mapcache_tile *tile); +MS_DLL_EXPORT void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt); +MS_DLL_EXPORT char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt); /** @} */ -mapcache_http_response* mapcache_core_get_capabilities(mapcache_context *ctx, mapcache_service *service, mapcache_request_get_capabilities *req_caps, char *url, char *path_info, mapcache_cfg *config); -mapcache_http_response* mapcache_core_get_tile(mapcache_context *ctx, mapcache_request_get_tile *req_tile); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_capabilities(mapcache_context *ctx, mapcache_service *service, mapcache_request_get_capabilities *req_caps, char *url, char *path_info, mapcache_cfg *config); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_tile(mapcache_context *ctx, mapcache_request_get_tile *req_tile); -mapcache_http_response* mapcache_core_get_map(mapcache_context *ctx, mapcache_request_get_map *req_map); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_map(mapcache_context *ctx, mapcache_request_get_map *req_map); -mapcache_http_response* mapcache_core_get_featureinfo(mapcache_context *ctx, mapcache_request_get_feature_info *req_fi); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_get_featureinfo(mapcache_context *ctx, mapcache_request_get_feature_info *req_fi); -mapcache_http_response* mapcache_core_proxy_request(mapcache_context *ctx, mapcache_request_proxy *req_proxy); -mapcache_http_response* mapcache_core_respond_to_error(mapcache_context *ctx); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_proxy_request(mapcache_context *ctx, mapcache_request_proxy *req_proxy); +MS_DLL_EXPORT mapcache_http_response* mapcache_core_respond_to_error(mapcache_context *ctx); /* in grid.c */ @@ -1699,8 +1283,11 @@ const char* mapcache_grid_get_crs(mapcache_context *ctx, mapcache_grid *grid); const char* mapcache_grid_get_srs(mapcache_context *ctx, mapcache_grid *grid); -void mapcache_grid_get_extent(mapcache_context *ctx, mapcache_grid *grid, +MS_DLL_EXPORT void mapcache_grid_get_tile_extent(mapcache_context *ctx, mapcache_grid *grid, int x, int y, int z, mapcache_extent *bbox); + +MS_DLL_EXPORT void mapcache_grid_get_metatile_extent(mapcache_context *ctx, mapcache_tile *tile, mapcache_extent *bbox); + /** * \brief compute x y value for given lon/lat (dx/dy) and given zoomlevel * @param ctx @@ -1711,7 +1298,7 @@ * @param x * @param y */ -void mapcache_grid_get_xy(mapcache_context *ctx, mapcache_grid *grid, double dx, double dy, int z, int *x, int *y); +MS_DLL_EXPORT void mapcache_grid_get_xy(mapcache_context *ctx, mapcache_grid *grid, double dx, double dy, int z, int *x, int *y); double mapcache_grid_get_resolution(mapcache_extent *bbox, int sx, int sy); double mapcache_grid_get_horizontal_resolution(mapcache_extent *bbox, int width); @@ -1731,15 +1318,18 @@ * \param extent * \param tolerance the number of tiles around the given extent that can be requested without returning an error. */ -void mapcache_grid_compute_limits(const mapcache_grid *grid, const mapcache_extent *extent, mapcache_extent_i *limits, int tolerance); +MS_DLL_EXPORT void mapcache_grid_compute_limits(const mapcache_grid *grid, const mapcache_extent *extent, mapcache_extent_i *limits, int tolerance); /* in util.c */ -int mapcache_util_extract_int_list(mapcache_context *ctx, const char* args, const char *sep, int **numbers, +MS_DLL_EXPORT int mapcache_util_extract_int_list(mapcache_context *ctx, const char* args, const char *sep, int **numbers, int *numbers_count); -int mapcache_util_extract_double_list(mapcache_context *ctx, const char* args, const char *sep, double **numbers, +MS_DLL_EXPORT int mapcache_util_extract_double_list(mapcache_context *ctx, const char* args, const char *sep, double **numbers, int *numbers_count); char *mapcache_util_str_replace(apr_pool_t *pool, const char *string, const char *substr, const char *replacement ); +void mapcache_util_quadkey_decode(mapcache_context *ctx, const char *quadkey, int *x, int *y, int *z); + +char* mapcache_util_quadkey_encode(mapcache_context *ctx, int x, int y, int z); /** * \brief replace dangerous characters in string @@ -1751,10 +1341,19 @@ */ char* mapcache_util_str_sanitize(apr_pool_t *pool, const char *str, const char* from, char to); +typedef enum { + MAPCACHE_UTIL_XML_SECTION_TEXT, + MAPCACHE_UTIL_XML_SECTION_ATTRIBUTE, + MAPCACHE_UTIL_XML_SECTION_COMMENT +} mapcache_util_xml_section_type; + +char* mapcache_util_str_xml_escape(apr_pool_t *pool, const char *str, mapcache_util_xml_section_type xml_section_type); + char* mapcache_util_get_tile_dimkey(mapcache_context *ctx, mapcache_tile *tile, char* sanitized_chars, char *sanitize_to); char* mapcache_util_get_tile_key(mapcache_context *ctx, mapcache_tile *tile, char *stemplate, char* sanitized_chars, char *sanitize_to); +void mapcache_make_parent_dirs(mapcache_context *ctx, char *filename); /**\defgroup imageio Image IO */ /** @{ */ @@ -1777,6 +1376,15 @@ MAPCACHE_PHOTOMETRIC_YCBCR } mapcache_photometric; +/** + * optimization settings (mostly) for jpeg + */ +typedef enum { + MAPCACHE_OPTIMIZE_NO, + MAPCACHE_OPTIMIZE_YES, + MAPCACHE_OPTIMIZE_ARITHMETIC +} mapcache_optimization; + /**\interface mapcache_image_format * \brief an image format * \sa mapcache_image_format_jpeg @@ -1815,13 +1423,14 @@ mapcache_image_format format; mapcache_image_format *transparent; mapcache_image_format *opaque; + unsigned int alpha_cutoff; /* default 255. pixels with alpha >= alpha_cutoff will be considered fully opaque */ }; mapcache_buffer* mapcache_empty_png_decode(mapcache_context *ctx, int width, int height, const unsigned char *hex_color, int *is_empty); mapcache_image_format* mapcache_imageio_create_mixed_format(apr_pool_t *pool, - char *name, mapcache_image_format *transparent, mapcache_image_format *opaque); + char *name, mapcache_image_format *transparent, mapcache_image_format *opaque, unsigned int alpha_cutoff); /**\class mapcache_image_format_png_q * \brief Quantized PNG format @@ -1884,10 +1493,11 @@ mapcache_image_format format; int quality; /**< JPEG quality, 1-100 */ mapcache_photometric photometric; + mapcache_optimization optimize; }; mapcache_image_format* mapcache_imageio_create_jpeg_format(apr_pool_t *pool, char *name, int quality, - mapcache_photometric photometric); + mapcache_photometric photometric, mapcache_optimization optimize); /** * @param r @@ -1936,13 +1546,19 @@ double resolution; } mapcache_interval; -typedef enum { - MAPCACHE_DIMENSION_VALUES, - MAPCACHE_DIMENSION_REGEX, - MAPCACHE_DIMENSION_INTERVALS, - MAPCACHE_DIMENSION_TIME, - MAPCACHE_DIMENSION_SQLITE -} mapcache_dimension_type; +struct mapcache_requested_dimension { + mapcache_dimension *dimension; + char *requested_value; + char *cached_value; +}; + +void mapcache_tile_set_cached_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value); +void mapcache_map_set_cached_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value); +void mapcache_tile_set_requested_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value); +void mapcache_map_set_requested_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value); +void mapcache_set_requested_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value); +void mapcache_set_cached_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value); +MS_DLL_EXPORT apr_array_header_t *mapcache_requested_dimensions_clone(apr_pool_t *pool, apr_array_header_t *src); struct mapcache_dimension { mapcache_dimension_type type; @@ -1950,24 +1566,24 @@ char *unit; apr_table_t *metadata; char *default_value; - int skip_validation; /** - * \brief validate the given value - * - * \param value is updated in case the given value is correct but has to be represented otherwise, - * e.g. to round off a value - * \returns MAPCACHE_SUCCESS if the given value is correct for the current dimension - * \returns MAPCACHE_FAILURE if not + * \brief return the list of dimension values that match the requested entry */ - int (*validate)(mapcache_context *context, mapcache_dimension *dimension, char **value); + apr_array_header_t* (*get_entries_for_value)(mapcache_context *ctx, mapcache_dimension *dimension, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid); /** - * \brief returns a list of values that are authorized for this dimension - * - * \returns a list of character strings that will be included in the capabilities element + * \brief return all possible values + */ + apr_array_header_t* (*get_all_entries)(mapcache_context *ctx, mapcache_dimension *dimension, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid); + + /** + * \brief return all possible values formatted in a way compatible with OGC capabilities element */ - apr_array_header_t* (*print_ogc_formatted_values)(mapcache_context *context, mapcache_dimension *dimension); + apr_array_header_t* (*get_all_ogc_formatted_entries)(mapcache_context *ctx, mapcache_dimension *dimension, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid); /** * \brief parse the value given in the configuration @@ -1975,79 +1591,10 @@ void (*configuration_parse_xml)(mapcache_context *context, mapcache_dimension *dim, ezxml_t node); }; -struct mapcache_dimension_values { - mapcache_dimension dimension; - int nvalues; - char **values; - int case_sensitive; -}; - -struct mapcache_dimension_sqlite { - mapcache_dimension dimension; - char *dbfile; - char *validate_query; - char *list_query; -}; - -struct mapcache_dimension_regex { - mapcache_dimension dimension; - char *regex_string; -#ifdef USE_PCRE - pcre *pcregex; -#else - regex_t *regex; -#endif -}; - -struct mapcache_dimension_intervals { - mapcache_dimension dimension; - int nintervals; - mapcache_interval *intervals; -}; - -struct mapcache_dimension_time { - mapcache_dimension dimension; - int nintervals; - mapcache_interval *intervals; -}; - -mapcache_dimension* mapcache_dimension_values_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_sqlite_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_regex_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_intervals_create(apr_pool_t *pool); -mapcache_dimension* mapcache_dimension_time_create(apr_pool_t *pool); - -typedef enum { - MAPCACHE_TIMEDIMENSION_ASSEMBLY_STACK, - MAPCACHE_TIMEDIMENSION_ASSEMBLY_ANIMATE -} mapcache_timedimension_assembly_type; - -typedef enum { - MAPCACHE_TIMEDIMENSION_SOURCE_SQLITE -} mapcache_timedimension_source_type; - -apr_array_header_t* mapcache_timedimension_get_entries_for_value(mapcache_context *ctx, mapcache_timedimension *timedimesnion, - mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, const char *value); - -struct mapcache_timedimension { - mapcache_timedimension_assembly_type assembly_type; - void (*configuration_parse_xml)(mapcache_context *context, mapcache_timedimension *dim, ezxml_t node); - apr_array_header_t* (*get_entries_for_interval)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset, - mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end); - apr_array_header_t* (*get_all_entries)(mapcache_context *ctx, mapcache_timedimension *dim, mapcache_tileset *tileset); - char *default_value; - char *key; /* TIME, hardcoded */ -}; - -#ifdef USE_SQLITE -typedef struct mapcache_timedimension_sqlite mapcache_timedimension_sqlite; -struct mapcache_timedimension_sqlite { - mapcache_timedimension timedimension; - char *dbfile; - char *query; -}; -mapcache_timedimension* mapcache_timedimension_sqlite_create(apr_pool_t *pool); -#endif +mapcache_dimension* mapcache_dimension_values_create(mapcache_context *ctx, apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_sqlite_create(mapcache_context *ctx, apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_regex_create(mapcache_context *ctx, apr_pool_t *pool); +mapcache_dimension* mapcache_dimension_time_create(mapcache_context *ctx, apr_pool_t *pool); int mapcache_is_axis_inverted(const char *srs); @@ -2060,10 +1607,10 @@ void *connection; }; -typedef void (*mapcache_connection_constructor)(mapcache_context *ctx, void **connection, void *params, apr_pool_t *process_pool); -typedef void (*mapcache_connection_destructor)(void *connection, apr_pool_t *process_pool); +typedef void (*mapcache_connection_constructor)(mapcache_context *ctx, void **connection, void *params); +typedef void (*mapcache_connection_destructor)(void *connection); -apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool); +MS_DLL_EXPORT apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool); mapcache_pooled_connection* mapcache_connection_pool_get_connection(mapcache_context *ctx, char *key, mapcache_connection_constructor constructor, mapcache_connection_destructor destructor, diff -Nru mapcache-1.4.0/include/mapcache_services.h mapcache-1.6.1/include/mapcache_services.h --- mapcache-1.4.0/include/mapcache_services.h 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/include/mapcache_services.h 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,162 @@ +/****************************************************************************** + * + * Project: MapCache + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2016 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + +/*! \file mapcache_services.h + \brief structure declarations for supported services + */ + + +#ifndef MAPCACHE_SERVICES_H_ +#define MAPCACHE_SERVICES_H_ + +#include "mapcache.h" + + +/** \addtogroup services */ +/** @{ */ + +typedef struct mapcache_request_get_capabilities_wmts mapcache_request_get_capabilities_wmts; +typedef struct mapcache_service_wmts mapcache_service_wmts; + +struct mapcache_request_get_capabilities_wmts { + mapcache_request_get_capabilities request; +}; + +/**\class mapcache_service_wmts + * \brief a WMTS service + * \implements mapcache_service + */ +struct mapcache_service_wmts { + mapcache_service service; +}; + + + +typedef struct mapcache_service_wms mapcache_service_wms; +typedef struct mapcache_request_get_capabilities_wms mapcache_request_get_capabilities_wms; + +struct mapcache_request_get_capabilities_wms { + mapcache_request_get_capabilities request; +}; + +/**\class mapcache_service_wms + * \brief an OGC WMS service + * \implements mapcache_service + */ +struct mapcache_service_wms { + mapcache_service service; + int maxsize; + apr_array_header_t *forwarding_rules; + mapcache_getmap_strategy getmap_strategy; + mapcache_resample_mode resample_mode; + mapcache_image_format *getmap_format; + int allow_format_override; /* can the client specify which image format should be returned */ +}; + +typedef struct mapcache_service_ve mapcache_service_ve; + +/**\class mapcache_service_ve + * \brief a virtualearth service + * \implements mapcache_service + */ +struct mapcache_service_ve { + mapcache_service service; +}; + + +typedef struct mapcache_service_gmaps mapcache_service_gmaps; +typedef struct mapcache_service_tms mapcache_service_tms; +typedef struct mapcache_request_get_capabilities_tms mapcache_request_get_capabilities_tms; + +/**\class mapcache_service_tms + * \brief a TMS service + * \implements mapcache_service + */ +struct mapcache_service_tms { + mapcache_service service; + int reverse_y; +}; + +struct mapcache_request_get_capabilities_tms { + mapcache_request_get_capabilities request; + mapcache_tileset *tileset; + mapcache_grid_link *grid_link; + char *version; +}; + + +typedef struct mapcache_service_mapguide mapcache_service_mapguide; + +struct mapcache_service_mapguide { + mapcache_service service; + int rows_per_folder; + int cols_per_folder; +}; + + + +typedef struct mapcache_service_kml mapcache_service_kml; +typedef struct mapcache_request_get_capabilities_kml mapcache_request_get_capabilities_kml; +struct mapcache_request_get_capabilities_kml { + mapcache_request_get_capabilities request; + mapcache_tile *tile; + mapcache_tileset *tileset; + mapcache_grid_link *grid; +}; +/**\class mapcache_service_kml + * \brief a KML superoverlay service + * \implements mapcache_service + */ +struct mapcache_service_kml { + mapcache_service service; +}; + +typedef struct mapcache_request_get_capabilities_demo mapcache_request_get_capabilities_demo; +typedef struct mapcache_service_demo mapcache_service_demo; + +/** + * the capabilities request for a specific service, to be able to create + * demo pages specific to a given service + */ +struct mapcache_request_get_capabilities_demo { + mapcache_request_get_capabilities request; + mapcache_service *service; +}; + +/**\class mapcache_service_demo + * \brief a demo service + * \implements mapcache_service + */ +struct mapcache_service_demo { + mapcache_service service; + +}; + + + + +#endif /*MAPCACHE_SERVICES_H*/ diff -Nru mapcache-1.4.0/include/util.h mapcache-1.6.1/include/util.h --- mapcache-1.4.0/include/util.h 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/include/util.h 2017-09-29 21:31:13.000000000 +0000 @@ -87,13 +87,19 @@ #endif #endif +#if defined(_WIN32) && !defined(__CYGWIN__) +# define MS_DLL_EXPORT __declspec(dllexport) +#else +#define MS_DLL_EXPORT +#endif #if defined(_WIN32) struct mctimeval { long tv_sec; /* seconds */ long tv_usec; /* and microseconds */ }; -void mapcache_gettimeofday(struct mctimeval *t, void *__not_used_here__); +MS_DLL_EXPORT void mapcache_gettimeofday(struct mctimeval *t, void *__not_used_here__); +MS_DLL_EXPORT char * strptime (const char *buf, const char *format, struct tm *timeptr); #else # include /* for gettimeofday() */ # define mctimeval timeval diff -Nru mapcache-1.4.0/lib/buffer.c mapcache-1.6.1/lib/buffer.c --- mapcache-1.4.0/lib/buffer.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/buffer.c 2017-09-29 21:31:13.000000000 +0000 @@ -57,7 +57,6 @@ mapcache_buffer *buffer = apr_pcalloc(pool, sizeof(mapcache_buffer)); if(!buffer) return NULL; buffer->pool = pool; - if(initialStorage <=0) initialStorage = 1; buffer->avail = initialStorage; if(buffer->avail) { buffer->buf = malloc(buffer->avail); diff -Nru mapcache-1.4.0/lib/cache_bdb.c mapcache-1.6.1/lib/cache_bdb.c --- mapcache-1.4.0/lib/cache_bdb.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_bdb.c 2017-09-29 21:31:13.000000000 +0000 @@ -1,7 +1,6 @@ /****************************************************************************** - * $Id$ * - * Project: MapServer + * Project: MapCache * Purpose: MapCache tile caching support file: Berkeley DB cache backend * Author: Thomas Bonfort and the MapServer team. * @@ -27,10 +26,8 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" -#ifdef USE_BDB - #include "mapcache.h" +#ifdef USE_BDB #include #include #include @@ -51,6 +48,13 @@ #define PAGESIZE 64*1024 #define CACHESIZE 1024*1024 +typedef struct mapcache_cache_bdb mapcache_cache_bdb; +struct mapcache_cache_bdb { + mapcache_cache cache; + char *basedir; + char *key_template; +}; + struct bdb_env { DB* db; DB_ENV *env; @@ -58,56 +62,60 @@ char *errmsg; }; -void mapcache_bdb_connection_constructor(mapcache_context *ctx, void **conn_, void *params, apr_pool_t *pool) +void mapcache_bdb_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; int env_flags; int mode; mapcache_cache_bdb *cache = (mapcache_cache_bdb*)params; - char *dbfile = apr_pstrcat(pool,cache->basedir,"/",cache->cache.name,".db",NULL); + char *dbfile = malloc(strlen(cache->basedir)+strlen(cache->cache.name)+strlen("/.db")+1); struct bdb_env *benv = calloc(1,sizeof(struct bdb_env)); + dbfile[0]=0; + strcat(strcat(strcat(strcat(dbfile,cache->basedir),"/"),cache->cache.name),".db"); /*yuk yuk*/ *conn_ = benv; ret = db_env_create(&benv->env, 0); if(ret) { ctx->set_error(ctx, 500, "bdb cache failure for db_env_create: %s", db_strerror(ret)); - free(benv); - return; + goto cleanup_error; } ret = benv->env->set_cachesize(benv->env,0,CACHESIZE,1); /* set a larger cache size than default */ if(ret) { ctx->set_error(ctx, 500, "bdb cache failure for db->set_cachesize: %s", db_strerror(ret)); - free(benv); - return; + goto cleanup_error; } env_flags = DB_INIT_CDB|DB_INIT_MPOOL|DB_CREATE; ret = benv->env->open(benv->env,cache->basedir,env_flags,0); if(ret) { ctx->set_error(ctx,500,"bdb cache failure for env->open: %s", db_strerror(ret)); - free(benv); - return; + goto cleanup_error; } if ((ret = db_create(&benv->db, benv->env, 0)) != 0) { ctx->set_error(ctx,500,"bdb cache failure for db_create: %s", db_strerror(ret)); - free(benv); + goto cleanup_error; } mode = DB_BTREE; ret = benv->db->set_pagesize(benv->db,PAGESIZE); /* set pagesize to maximum allowed, as tile data is usually pretty large */ if(ret) { ctx->set_error(ctx,500,"bdb cache failure for db->set_pagesize: %s", db_strerror(ret)); - free(benv); - return; + goto cleanup_error; } if ((ret = benv->db->open(benv->db, NULL, dbfile, NULL, mode, DB_CREATE, 0664)) != 0) { ctx->set_error(ctx,500,"bdb cache failure 1 for db->open: %s", db_strerror(ret)); - free(benv); - return; + goto cleanup_error; } + + goto cleanup; + +cleanup_error: + free(benv); +cleanup: + free(dbfile); } -void mapcache_bdb_connection_destructor(void *conn_, apr_pool_t *pool) +void mapcache_bdb_connection_destructor(void *conn_) { struct bdb_env *benv = (struct bdb_env*)conn_; benv->db->close(benv->db,0); @@ -144,7 +152,7 @@ mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); mapcache_pooled_connection *pc; - struct bdb_env *benv; + struct bdb_env *benv; pc = _bdb_get_conn(ctx,cache,tile,1); if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; benv = pc->connection; @@ -173,7 +181,7 @@ mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); mapcache_pooled_connection *pc; - struct bdb_env *benv; + struct bdb_env *benv; pc = _bdb_get_conn(ctx,cache,tile,0); GC_CHECK_ERROR(ctx); benv = pc->connection; @@ -195,13 +203,14 @@ { DBT key,data; int ret; + char *skey; mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; mapcache_pooled_connection *pc; - struct bdb_env *benv; + struct bdb_env *benv; pc = _bdb_get_conn(ctx,cache,tile,1); if(GC_HAS_ERROR(ctx)) return MAPCACHE_FALSE; benv = pc->connection; - char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); + skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); data.flags = DB_DBT_MALLOC; @@ -242,7 +251,7 @@ mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; char *skey = mapcache_util_get_tile_key(ctx,tile,cache->key_template,NULL,NULL); mapcache_pooled_connection *pc; - struct bdb_env *benv; + struct bdb_env *benv; now = apr_time_now(); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); @@ -254,7 +263,7 @@ tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); GC_CHECK_ERROR(ctx); } - + if(tile->raw_image->h==256 && tile->raw_image->w==256 && mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { data.size = 5+sizeof(apr_time_t); data.data = apr_palloc(ctx->pool,data.size); @@ -271,7 +280,7 @@ data.size = tile->encoded_data->size; tile->encoded_data->size -= sizeof(apr_time_t); } - + pc = _bdb_get_conn(ctx,cache,tile,0); GC_CHECK_ERROR(ctx); benv = pc->connection; @@ -295,7 +304,7 @@ apr_time_t now; mapcache_cache_bdb *cache = (mapcache_cache_bdb*)pcache; mapcache_pooled_connection *pc; - struct bdb_env *benv; + struct bdb_env *benv; now = apr_time_now(); memset(&key, 0, sizeof(DBT)); memset(&data, 0, sizeof(DBT)); @@ -303,7 +312,7 @@ pc = _bdb_get_conn(ctx,cache,&tiles[0],0); GC_CHECK_ERROR(ctx); benv = pc->connection; - + for(i=0; icache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_BDB; - cache->cache.tile_delete = _mapcache_cache_bdb_delete; - cache->cache.tile_get = _mapcache_cache_bdb_get; - cache->cache.tile_exists = _mapcache_cache_bdb_has_tile; - cache->cache.tile_set = _mapcache_cache_bdb_set; - cache->cache.tile_multi_set = _mapcache_cache_bdb_multiset; + cache->cache._tile_delete = _mapcache_cache_bdb_delete; + cache->cache._tile_get = _mapcache_cache_bdb_get; + cache->cache._tile_exists = _mapcache_cache_bdb_has_tile; + cache->cache._tile_set = _mapcache_cache_bdb_set; + cache->cache._tile_multi_set = _mapcache_cache_bdb_multiset; cache->cache.configuration_post_config = _mapcache_cache_bdb_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_bdb_configuration_parse_xml; cache->basedir = NULL; @@ -413,6 +422,11 @@ return (mapcache_cache*)cache; } +#else +mapcache_cache* mapcache_cache_bdb_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"BERKELEYDB support not compiled in this version"); + return NULL; +} #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.4.0/lib/cache.c mapcache-1.6.1/lib/cache.c --- mapcache-1.4.0/lib/cache.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/cache.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,159 @@ +/****************************************************************************** + * + * Project: MapServer + * Purpose: MapCache tile caching: generic cache access + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2015 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without compositeriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ +#include "mapcache.h" +#include + +int mapcache_cache_tile_get(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i,rv; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_get on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) get retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_get(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } + return rv; +} + +void mapcache_cache_tile_delete(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_delete on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + if(tile->tileset->read_only) + return; + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) delete retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_delete(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } +} + +int mapcache_cache_tile_exists(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i,rv; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_exists on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) exists retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_exists(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } + return rv; +} + +void mapcache_cache_tile_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tile) { + int i; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_set on cache (%s): (tileset=%s, grid=%s, z=%d, x=%d, y=%d",cache->name,tile->tileset->name,tile->grid_link->grid->name,tile->z,tile->x, tile->y); +#endif + if(tile->tileset->read_only) + return; + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) set retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_set(ctx,cache,tile); + if(!GC_HAS_ERROR(ctx)) + break; + } +} + +void mapcache_cache_tile_multi_set(mapcache_context *ctx, mapcache_cache *cache, mapcache_tile *tiles, int ntiles) { + int i; +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"calling tile_multi_set on cache (%s): (tileset=%s, grid=%s, first tile: z=%d, x=%d, y=%d",cache->name,tiles[0].tileset->name,tiles[0].grid_link->grid->name, + tiles[0].z,tiles[0].x, tiles[0].y); +#endif + if((&tiles[0])->tileset->read_only) + return; + if(cache->_tile_multi_set) { + for(i=0;i<=cache->retry_count;i++) { + if(i) { + ctx->log(ctx,MAPCACHE_INFO,"cache (%s) multi-set retry %d of %d. previous try returned error: %s",cache->name,i,cache->retry_count,ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(cache->retry_delay > 0) { + double wait = cache->retry_delay; + int j = 0; + for(j=1;j_tile_multi_set(ctx,cache,tiles,ntiles); + if(!GC_HAS_ERROR(ctx)) + break; + } + } else { + for( i=0;i + +typedef struct mapcache_cache_composite mapcache_cache_composite; + +typedef struct mapcache_cache_composite_cache_link mapcache_cache_composite_cache_link; +struct mapcache_cache_composite_cache_link { + mapcache_cache *cache; + int minzoom; + int maxzoom; + apr_array_header_t *grids; + apr_table_t *dimensions; /* key/value pairs of dimensions */ +}; + +struct mapcache_cache_composite { + mapcache_cache cache; + apr_array_header_t *cache_links; +}; static mapcache_cache_composite_cache_link* _mapcache_cache_link_create(apr_pool_t *pool) { mapcache_cache_composite_cache_link *cl = apr_pcalloc(pool, sizeof(mapcache_cache_composite_cache_link)); @@ -59,11 +76,31 @@ /* not found */ if(j == cache_link->grids->nelts) continue; } + if(cache_link->dimensions) { + const apr_array_header_t *array = apr_table_elts(cache_link->dimensions); + apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; + int j; + if(!tile->dimensions) continue; /* the cache link refers to dimensions, but this tile does not have any, it cannot match */ + + for (j = 0; j < array->nelts; j++) { + char *dim = elts[j].key; + char *dimval = elts[j].val; + int k; + for(k=0;kdimensions->nelts;k++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + if(!strcmp(rdim->dimension->name,dim) && !strcmp(rdim->cached_value,dimval)) + break; + } + if(k == tile->dimensions->nelts) break; /* no tile dimension matched the current cache dimension */ + } + if(j != array->nelts) continue; /* we broke out early from the cache dimension loop, so at least one was not correct */ + } return cache_link->cache; } ctx->set_error(ctx, 500, "no cache matches for given tile request"); return NULL; } + static int _mapcache_cache_composite_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { mapcache_cache_composite *cache = (mapcache_cache_composite*)pcache; @@ -71,7 +108,7 @@ subcache = _mapcache_composite_cache_get(ctx, cache, tile); if(GC_HAS_ERROR(ctx) || !subcache) return MAPCACHE_FAILURE; - return subcache->tile_exists(ctx, subcache, tile); + return mapcache_cache_tile_exists(ctx, subcache, tile); } static void _mapcache_cache_composite_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) @@ -81,7 +118,7 @@ subcache = _mapcache_composite_cache_get(ctx, cache, tile); GC_CHECK_ERROR(ctx); /*delete the tile itself*/ - subcache->tile_delete(ctx,subcache,tile); + mapcache_cache_tile_delete(ctx,subcache,tile); } /** @@ -97,7 +134,7 @@ mapcache_cache *subcache; subcache = _mapcache_composite_cache_get(ctx, cache, tile); GC_CHECK_ERROR_RETURN(ctx); - return subcache->tile_get(ctx,subcache,tile); + return mapcache_cache_tile_get(ctx,subcache,tile); } static void _mapcache_cache_composite_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) @@ -106,7 +143,7 @@ mapcache_cache *subcache; subcache = _mapcache_composite_cache_get(ctx, cache, tile); GC_CHECK_ERROR(ctx); - return subcache->tile_set(ctx,subcache,tile); + return mapcache_cache_tile_set(ctx,subcache,tile); } static void _mapcache_cache_composite_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) @@ -115,14 +152,7 @@ mapcache_cache *subcache; subcache = _mapcache_composite_cache_get(ctx, cache, &tiles[0]); GC_CHECK_ERROR(ctx); - if(subcache->tile_multi_set) { - return subcache->tile_multi_set(ctx,subcache,tiles,ntiles); - } else { - int i; - for(i=0; itile_set(ctx, subcache, &tiles[i]); - } - } + return mapcache_cache_tile_multi_set(ctx,subcache,tiles,ntiles); } /** @@ -168,6 +198,37 @@ } cachelink->minzoom = zoom; } + sZoom = (char*)ezxml_attr(cur_node,"grids"); + if(sZoom) { + char *grids = apr_pstrdup(ctx->pool,sZoom),*key,*last; + for(key = apr_strtok(grids, ",", &last); key; key = apr_strtok(NULL,",",&last)) { + /*loop through grids*/ + if(!cachelink->grids) { + cachelink->grids =apr_array_make(ctx->pool,1,sizeof(char*)); + } + APR_ARRAY_PUSH(cachelink->grids,char*) = key; + } + } + sZoom = (char*)ezxml_attr(cur_node,"dimensions"); + if(sZoom) { + char *dims = apr_pstrdup(ctx->pool,sZoom),*key,*last; + for(key = apr_strtok(dims, ",", &last); key; key = apr_strtok(NULL,",",&last)) { + char *dimname; + /*loop through dims*/ + if(!cachelink->dimensions) { + cachelink->dimensions =apr_table_make(ctx->pool,1); + } + dimname = key; + while(*key && *key!='=') key++; + if(!(*key)) { + ctx->set_error(ctx,400,"failed to parse composite cache dimensions. expecting dimensions=\"dim1=val1,dim2=val2\""); + return; + } + *key = 0; + key++; + apr_table_set(cachelink->dimensions,dimname,key); + } + } APR_ARRAY_PUSH(cache->cache_links,mapcache_cache_composite_cache_link*) = cachelink; } @@ -194,11 +255,11 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_COMPOSITE; - cache->cache.tile_delete = _mapcache_cache_composite_tile_delete; - cache->cache.tile_get = _mapcache_cache_composite_tile_get; - cache->cache.tile_exists = _mapcache_cache_composite_tile_exists; - cache->cache.tile_set = _mapcache_cache_composite_tile_set; - cache->cache.tile_multi_set = _mapcache_cache_composite_tile_multi_set; + cache->cache._tile_delete = _mapcache_cache_composite_tile_delete; + cache->cache._tile_get = _mapcache_cache_composite_tile_get; + cache->cache._tile_exists = _mapcache_cache_composite_tile_exists; + cache->cache._tile_set = _mapcache_cache_composite_tile_set; + cache->cache._tile_multi_set = _mapcache_cache_composite_tile_multi_set; cache->cache.configuration_post_config = _mapcache_cache_composite_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_composite_configuration_parse_xml; return (mapcache_cache*)cache; diff -Nru mapcache-1.4.0/lib/cache_couchbase.c mapcache-1.6.1/lib/cache_couchbase.c --- mapcache-1.4.0/lib/cache_couchbase.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_couchbase.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,13 +27,31 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" + #ifdef USE_COUCHBASE -#include "mapcache.h" #include #include #include +#include + +typedef struct mapcache_cache_couchbase mapcache_cache_couchbase; + +/**\class mapcache_cache_couchbase + * \brief a mapcache_cache on couchbase servers + * \implements mapcache_cache + */ +struct mapcache_cache_couchbase { + mapcache_cache cache; +// apr_reslist_t *connection_pool; + char *host; + char *username; + char *password; + char *bucket; + mapcache_context *ctx; +}; + typedef struct getStruct { @@ -456,6 +474,11 @@ return (mapcache_cache*)cache; } +#else +mapcache_cache* mapcache_cache_couchbase_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"COUCHBASE support not compiled in this version"); + return NULL; +} #endif /* vim: ai ts=3 sts=3 et sw=3 diff -Nru mapcache-1.4.0/lib/cache_disk.c mapcache-1.6.1/lib/cache_disk.c --- mapcache-1.4.0/lib/cache_disk.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_disk.c 2017-09-29 21:31:13.000000000 +0000 @@ -39,6 +39,26 @@ #include #endif +/**\class mapcache_cache_disk + * \brief a mapcache_cache on a filesytem + * \implements mapcache_cache + */ +typedef struct mapcache_cache_disk mapcache_cache_disk; +struct mapcache_cache_disk { + mapcache_cache cache; + char *base_directory; + char *filename_template; + int symlink_blank; + int creation_retry; + + /** + * Set filename for a given tile + * \memberof mapcache_cache_disk + */ + void (*tile_key)(mapcache_context *ctx, mapcache_cache_disk *cache, mapcache_tile *tile, char **path); +}; + + /** * \brief computes the relative path between two destinations * @@ -119,11 +139,15 @@ tile->grid_link->grid->name, NULL); if(tile->dimensions) { - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->val,"/.",'#'); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + char *dimval; + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not set",entry->dimension->name); + return; + } + dimval = mapcache_util_str_sanitize(ctx->pool,entry->cached_value,"/.",'#'); *path = apr_pstrcat(ctx->pool,*path,"/",dimval,NULL); } } @@ -196,14 +220,20 @@ *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_z}", apr_psprintf(ctx->pool,"%d", tile->grid_link->grid->nlevels - tile->z - 1)); - if(tile->dimensions) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - char *dimval = apr_pstrdup(ctx->pool,entry->val); - char *iter = dimval; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i, mapcache_requested_dimension*); + char *dimval; + char *single_dim; + char *iter; + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not set",entry->dimension->name); + return; + } + dimval = apr_pstrdup(ctx->pool,entry->cached_value); + iter = dimval; while(*iter) { /* replace dangerous characters by '#' */ if(*iter == '.' || *iter == '/') { @@ -211,7 +241,11 @@ } iter++; } - dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->key,"#",dimval,NULL); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",dimval,NULL); + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(*path,single_dim)) { + *path = mapcache_util_str_replace(ctx->pool,*path, single_dim, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -251,14 +285,20 @@ *path = mapcache_util_str_replace(ctx->pool,*path, "{inv_z}", apr_psprintf(ctx->pool,"%d", tile->grid_link->grid->nlevels - tile->z - 1)); - if(tile->dimensions) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - char *dimval = apr_pstrdup(ctx->pool,entry->val); - char *iter = dimval; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i, mapcache_requested_dimension*); + char *dimval; + char *single_dim; + char *iter; + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not set",entry->dimension->name); + return; + } + dimval = apr_pstrdup(ctx->pool,entry->cached_value); + iter = dimval; while(*iter) { /* replace dangerous characters by '#' */ if(*iter == '.' || *iter == '/') { @@ -266,7 +306,11 @@ } iter++; } - dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->key,"#",dimval,NULL); + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",dimval,NULL); + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(*path,single_dim)) { + *path = mapcache_util_str_replace(ctx->pool,*path, single_dim, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -377,8 +421,8 @@ ctx->pool)) == APR_SUCCESS) { rv = apr_file_info_get(&finfo, APR_FINFO_SIZE|APR_FINFO_MTIME, f); if(!finfo.size) { - ctx->set_error(ctx, 500, "tile %s has no data",filename); - return MAPCACHE_FAILURE; + ctx->log(ctx, MAPCACHE_WARN, "tile %s has 0 length data",filename); + return MAPCACHE_CACHE_MISS; } size = finfo.size; @@ -391,10 +435,10 @@ * i.e. normally only once. */ tile->mtime = finfo.mtime; - tile->encoded_data = mapcache_buffer_create(size,ctx->pool); #ifndef NOMMAP + tile->encoded_data = mapcache_buffer_create(0,ctx->pool); rv = apr_mmap_create(&tilemmap,f,0,finfo.size,APR_MMAP_READ,ctx->pool); if(rv != APR_SUCCESS) { char errmsg[120]; @@ -404,6 +448,7 @@ tile->encoded_data->buf = tilemmap->mm; tile->encoded_data->size = tile->encoded_data->avail = finfo.size; #else + tile->encoded_data = mapcache_buffer_create(size,ctx->pool); //manually add the data to our buffer apr_file_read(f,(void*)tile->encoded_data->buf,&size); tile->encoded_data->size = size; @@ -442,7 +487,7 @@ apr_file_t *f; apr_status_t ret; char errmsg[120]; - char *filename, *hackptr1, *hackptr2=NULL; + char *filename; mapcache_cache_disk *cache = (mapcache_cache_disk*)pcache; const int creation_retry = cache->creation_retry; int retry_count_create_file = 0; @@ -462,26 +507,8 @@ cache->tile_key(ctx, cache, tile, &filename); GC_CHECK_ERROR(ctx); - /* find the location of the last '/' in the string */ - hackptr1 = filename; - while(*hackptr1) { - if(*hackptr1 == '/') - hackptr2 = hackptr1; - hackptr1++; - } - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - /* - * apr_dir_make_recursive sometimes sends back this error, although it should not. - * ignore this one - */ - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; - } - } - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); ret = apr_file_remove(filename,ctx->pool); if(ret != APR_SUCCESS && !APR_STATUS_IS_ENOENT(ret)) { @@ -497,26 +524,19 @@ } if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { char *blankname; + int retry_count_create_symlink = 0; + char *blankname_rel = NULL; _mapcache_cache_disk_blank_tile_key(ctx,cache,tile,tile->raw_image->data,&blankname); if(apr_file_open(&f, blankname, APR_FOPEN_READ, APR_OS_DEFAULT, ctx->pool) != APR_SUCCESS) { + /* create the blank file */ int isLocked; void *lock; if(!tile->encoded_data) { tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); GC_CHECK_ERROR(ctx); } - /* create the blank file */ - char *blankdirname = apr_psprintf(ctx->pool, "%s/%s/%s/blanks", - cache->base_directory, - tile->tileset->name, - tile->grid_link->grid->name); - if(APR_SUCCESS != (ret = apr_dir_make_recursive( - blankdirname, APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create directory %s for blank tiles",blankdirname, apr_strerror(ret,errmsg,120)); - return; - } - } + mapcache_make_parent_dirs(ctx,blankname); + GC_CHECK_ERROR(ctx); /* aquire a lock on the blank file */ isLocked = mapcache_lock_or_wait_for_resource(ctx,ctx->config->locker,blankname, &lock); @@ -527,7 +547,7 @@ APR_FOPEN_CREATE|APR_FOPEN_WRITE|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY, APR_OS_DEFAULT, ctx->pool)) != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to create file %s: %s",blankname, apr_strerror(ret,errmsg,120)); - mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); return; /* we could not create the file */ } @@ -535,17 +555,17 @@ ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",blankname, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120)); - mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); return; /* we could not create the file */ } if(bytes != tile->encoded_data->size) { ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", blankname, (int)bytes, (int)tile->encoded_data->size); - mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); return; } apr_file_close(f); - mapcache_unlock_resource(ctx,ctx->config->locker,blankname, lock); + mapcache_unlock_resource(ctx,ctx->config->locker, lock); #ifdef DEBUG ctx->log(ctx,MAPCACHE_DEBUG,"created blank tile %s",blankname); #endif @@ -554,12 +574,10 @@ apr_file_close(f); } - int retry_count_create_symlink = 0; /* * compute the relative path between tile and blank tile */ - char *blankname_rel = NULL; blankname_rel = relative_path(ctx,filename, blankname); GC_CHECK_ERROR(ctx); @@ -576,17 +594,8 @@ ctx->set_error(ctx, 500, "failed to link tile %s to %s: %s",filename, blankname_rel, error); return; /* we could not create the file */ } - - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create symlink, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ - } - } - - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); } #ifdef DEBUG ctx->log(ctx, MAPCACHE_DEBUG, "linked blank tile %s to %s",filename,blankname); @@ -603,6 +612,12 @@ GC_CHECK_ERROR(ctx); } + bytes = (apr_size_t)tile->encoded_data->size; + if(bytes == 0) { + ctx->set_error(ctx, 500, "attempting to write 0 length tile to %s",filename); + return; /* we could not create the file */ + } + /* * depending on configuration file creation will retry if it fails. * this can happen on nfs mounted network storage. @@ -618,33 +633,24 @@ ctx->set_error(ctx, 500, "failed to create file %s: %s",filename, apr_strerror(ret,errmsg,120)); return; /* we could not create the file */ } - - *hackptr2 = '\0'; - - if(APR_SUCCESS != (ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - if(!APR_STATUS_IS_EEXIST(ret)) { - ctx->set_error(ctx, 500, "failed to create file, can not create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ - } - } - - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); } - bytes = (apr_size_t)tile->encoded_data->size; ret = apr_file_write(f,(void*)tile->encoded_data->buf,&bytes); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to write data to file %s (wrote %d of %d bytes): %s",filename, (int)bytes, (int)tile->encoded_data->size, apr_strerror(ret,errmsg,120)); return; /* we could not create the file */ } - if(bytes != tile->encoded_data->size) { - ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", filename, (int)bytes, (int)tile->encoded_data->size); - } ret = apr_file_close(f); if(ret != APR_SUCCESS) { ctx->set_error(ctx, 500, "failed to close file %s:%s",filename, apr_strerror(ret,errmsg,120)); - return; /* we could not create the file */ + } + + if(bytes != tile->encoded_data->size) { + ctx->set_error(ctx, 500, "failed to write image data to %s, wrote %d of %d bytes", filename, (int)bytes, (int)tile->encoded_data->size); + apr_file_remove(filename, ctx->pool); } } @@ -729,10 +735,10 @@ cache->creation_retry = 0; cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_DISK; - cache->cache.tile_delete = _mapcache_cache_disk_delete; - cache->cache.tile_get = _mapcache_cache_disk_get; - cache->cache.tile_exists = _mapcache_cache_disk_has_tile; - cache->cache.tile_set = _mapcache_cache_disk_set; + cache->cache._tile_delete = _mapcache_cache_disk_delete; + cache->cache._tile_get = _mapcache_cache_disk_get; + cache->cache._tile_exists = _mapcache_cache_disk_has_tile; + cache->cache._tile_set = _mapcache_cache_disk_set; cache->cache.configuration_post_config = _mapcache_cache_disk_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_disk_configuration_parse_xml; return (mapcache_cache*)cache; diff -Nru mapcache-1.4.0/lib/cache_fallback.c mapcache-1.6.1/lib/cache_fallback.c --- mapcache-1.4.0/lib/cache_fallback.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_fallback.c 2017-09-29 21:31:13.000000000 +0000 @@ -28,11 +28,18 @@ #include "mapcache.h" +typedef struct mapcache_cache_fallback mapcache_cache_fallback; + +struct mapcache_cache_fallback { + mapcache_cache cache; + apr_array_header_t *caches; +}; + static int _mapcache_cache_fallback_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); - return subcache->tile_exists(ctx, subcache, tile); + return mapcache_cache_tile_exists(ctx, subcache, tile); } static void _mapcache_cache_fallback_tile_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) @@ -41,7 +48,7 @@ int i; for(i=0; icaches->nelts; i++) { mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - subcache->tile_delete(ctx, subcache, tile); + mapcache_cache_tile_delete(ctx, subcache, tile); ctx->clear_errors(ctx); /* ignore errors */ } } @@ -59,18 +66,18 @@ mapcache_cache *subcache; int i,ret; subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); - ret = subcache->tile_get(ctx, subcache, tile); + ret = mapcache_cache_tile_get(ctx, subcache, tile); if(ret == MAPCACHE_FAILURE) { int first_error = ctx->get_error(ctx); char *first_error_message = ctx->get_error_message(ctx); - ctx->log(ctx,MAPCACHE_WARN,"failed \"GET\" on primary cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Falling back on secondary caches", + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"GET\" on primary cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Falling back on secondary caches", APR_ARRAY_IDX(cache->caches,0,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); ctx->clear_errors(ctx); for(i=1; icaches->nelts; i++) { subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - if((ret = subcache->tile_get(ctx, subcache, tile)) == MAPCACHE_FAILURE) { - ctx->log(ctx,MAPCACHE_WARN,"failed \"GET\" on fallback cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Continuing with other fallback caches if available", + if((ret = mapcache_cache_tile_get(ctx, subcache, tile)) == MAPCACHE_FAILURE) { + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"GET\" on fallback cache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\". Continuing with other fallback caches if available", APR_ARRAY_IDX(cache->caches,0,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); ctx->clear_errors(ctx); continue; @@ -90,24 +97,22 @@ static void _mapcache_cache_fallback_tile_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; - int i,oneok=0,first_error=0; + int i,first_error=0; char *first_error_message; for(i=0; icaches->nelts; i++) { mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - subcache->tile_set(ctx, subcache, tile); + mapcache_cache_tile_set(ctx, subcache, tile); if(GC_HAS_ERROR(ctx)) { if(!first_error) { first_error = ctx->get_error(ctx); first_error_message = ctx->get_error_message(ctx); } - ctx->log(ctx,MAPCACHE_WARN,"failed \"SET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"SET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", APR_ARRAY_IDX(cache->caches,i,mapcache_cache*)->name,tile->z,tile->x,tile->y,tile->tileset->name); ctx->clear_errors(ctx); - } else { - oneok = 1; } } - if(!oneok) { + if(first_error) { ctx->set_error(ctx,first_error,first_error_message); } } @@ -115,34 +120,22 @@ static void _mapcache_cache_fallback_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { mapcache_cache_fallback *cache = (mapcache_cache_fallback*)pcache; - int i,oneok=0,first_error=0; + int i,first_error=0; char *first_error_message; for(i=0; icaches->nelts; i++) { mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - if(subcache->tile_multi_set) { - subcache->tile_multi_set(ctx, subcache, tiles, ntiles); - } - else { - int j; - for(j=0;jtile_set(ctx,subcache,&tiles[j]); - if(GC_HAS_ERROR(ctx)) - break; - } - } + mapcache_cache_tile_multi_set(ctx, subcache, tiles, ntiles); if(GC_HAS_ERROR(ctx)) { if(!first_error) { first_error = ctx->get_error(ctx); first_error_message = ctx->get_error_message(ctx); } - ctx->log(ctx,MAPCACHE_WARN,"failed \"MULTISET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", + ctx->log(ctx,MAPCACHE_DEBUG,"failed \"MULTISET\" on subcache \"%s\" for tile (z=%d,x=%d,y=%d) of tileset \"%s\"", APR_ARRAY_IDX(cache->caches,i,mapcache_cache*)->name,tiles[0].z,tiles[0].x,tiles[0].y,tiles[0].tileset->name); ctx->clear_errors(ctx); - } else { - oneok = 1; } } - if(!oneok) { + if(first_error) { ctx->set_error(ctx,first_error,first_error_message); } } @@ -190,11 +183,11 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_COMPOSITE; - cache->cache.tile_delete = _mapcache_cache_fallback_tile_delete; - cache->cache.tile_get = _mapcache_cache_fallback_tile_get; - cache->cache.tile_exists = _mapcache_cache_fallback_tile_exists; - cache->cache.tile_set = _mapcache_cache_fallback_tile_set; - cache->cache.tile_multi_set = _mapcache_cache_fallback_tile_multi_set; + cache->cache._tile_delete = _mapcache_cache_fallback_tile_delete; + cache->cache._tile_get = _mapcache_cache_fallback_tile_get; + cache->cache._tile_exists = _mapcache_cache_fallback_tile_exists; + cache->cache._tile_set = _mapcache_cache_fallback_tile_set; + cache->cache._tile_multi_set = _mapcache_cache_fallback_tile_multi_set; cache->cache.configuration_post_config = _mapcache_cache_fallback_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_fallback_configuration_parse_xml; return (mapcache_cache*)cache; diff -Nru mapcache-1.4.0/lib/cache_memcache.c mapcache-1.6.1/lib/cache_memcache.c --- mapcache-1.4.0/lib/cache_memcache.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_memcache.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,10 +27,29 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_MEMCACHE -#include "mapcache.h" +#include + +typedef struct mapcache_cache_memcache mapcache_cache_memcache; +/**\class mapcache_cache_memcache + * \brief a mapcache_cache on memcached servers + * \implements mapcache_cache + */ + +struct mapcache_cache_memcache_server { + char* host; + int port; +}; + +struct mapcache_cache_memcache { + mapcache_cache cache; + int nservers; + struct mapcache_cache_memcache_server *servers; + int detect_blank; +}; + struct mapcache_memcache_conn_param { mapcache_cache_memcache *cache; }; @@ -40,13 +59,13 @@ apr_pool_t *pool; }; -void mapcache_memcache_connection_constructor(mapcache_context *ctx, void **conn_, void *params, apr_pool_t *process_pool) { +void mapcache_memcache_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { struct mapcache_memcache_conn_param *param = params; mapcache_cache_memcache *cache = param->cache; struct mapcache_memcache_pooled_connection *pc; int i; pc = calloc(1,sizeof(struct mapcache_memcache_pooled_connection)); - apr_pool_create(&pc->pool,process_pool); + apr_pool_create(&pc->pool,NULL); if(APR_SUCCESS != apr_memcache_create(pc->pool, cache->nservers, 0, &(pc->memcache))) { ctx->set_error(ctx,500,"cache %s: failed to create memcache backend", cache->cache.name); return; @@ -65,7 +84,7 @@ *conn_ = pc; } -void mapcache_memcache_connection_destructor(void *conn_, apr_pool_t *process_pool) { +void mapcache_memcache_connection_destructor(void *conn_) { struct mapcache_memcache_pooled_connection *pc = conn_; apr_pool_destroy(pc->pool); free(pc); @@ -186,7 +205,7 @@ &(((char*)encoded_data->buf)[encoded_data->size-sizeof(apr_time_t)]), sizeof(apr_time_t)); - ((char*)encoded_data->buf)[encoded_data->size+sizeof(apr_time_t)]='\0'; + ((char*)encoded_data->buf)[encoded_data->size-sizeof(apr_time_t)]='\0'; encoded_data->avail = encoded_data->size; encoded_data->size -= sizeof(apr_time_t); if(((char*)encoded_data->buf)[0] == '#' && encoded_data->size > 1) { @@ -213,10 +232,11 @@ */ static void _mapcache_cache_memcache_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - char *key; + char *key, *data; int rv; /* set no expiration if not configured */ int expires =0; + apr_time_t now; mapcache_buffer *encoded_data = NULL; mapcache_cache_memcache *cache = (mapcache_cache_memcache*)pcache; mapcache_pooled_connection *pc; @@ -252,8 +272,8 @@ /* concatenate the current time to the end of the memcache data so we can extract it out * when we re-get the tile */ - char *data = calloc(1,encoded_data->size+sizeof(apr_time_t)); - apr_time_t now = apr_time_now(); + data = calloc(1,encoded_data->size+sizeof(apr_time_t)); + now = apr_time_now(); apr_pool_cleanup_register(ctx->pool, data, (void*)free, apr_pool_cleanup_null); memcpy(data,encoded_data->buf,encoded_data->size); memcpy(&(data[encoded_data->size]),&now,sizeof(apr_time_t)); @@ -344,15 +364,20 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_MEMCACHE; - cache->cache.tile_get = _mapcache_cache_memcache_get; - cache->cache.tile_exists = _mapcache_cache_memcache_has_tile; - cache->cache.tile_set = _mapcache_cache_memcache_set; - cache->cache.tile_delete = _mapcache_cache_memcache_delete; + cache->cache._tile_get = _mapcache_cache_memcache_get; + cache->cache._tile_exists = _mapcache_cache_memcache_has_tile; + cache->cache._tile_set = _mapcache_cache_memcache_set; + cache->cache._tile_delete = _mapcache_cache_memcache_delete; cache->cache.configuration_post_config = _mapcache_cache_memcache_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_memcache_configuration_parse_xml; return (mapcache_cache*)cache; } +#else +mapcache_cache* mapcache_cache_memcache_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"MEMCACHE support not compiled in this version"); + return NULL; +} #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.4.0/lib/cache_multitier.c mapcache-1.6.1/lib/cache_multitier.c --- mapcache-1.4.0/lib/cache_multitier.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_multitier.c 2017-09-29 21:31:13.000000000 +0000 @@ -28,13 +28,21 @@ #include "mapcache.h" +typedef struct mapcache_cache_multitier mapcache_cache_multitier; + +struct mapcache_cache_multitier { + mapcache_cache cache; + apr_array_header_t *caches; +}; + + static int _mapcache_cache_multitier_tile_exists(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; int i; for(i=0; icaches->nelts; i++) { mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - if(subcache->tile_exists(ctx, subcache, tile) == MAPCACHE_TRUE) { + if(mapcache_cache_tile_exists(ctx, subcache, tile) == MAPCACHE_TRUE) { return MAPCACHE_TRUE; } } @@ -47,7 +55,7 @@ int i; for(i=0; icaches->nelts; i++) { mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - subcache->tile_delete(ctx, subcache, tile); + mapcache_cache_tile_delete(ctx, subcache, tile); ctx->clear_errors(ctx); /* ignore errors */ } } @@ -65,16 +73,16 @@ mapcache_cache *subcache; int i,ret; subcache = APR_ARRAY_IDX(cache->caches,0,mapcache_cache*); - ret = subcache->tile_get(ctx, subcache, tile); + ret = mapcache_cache_tile_get(ctx, subcache, tile); if(ret == MAPCACHE_CACHE_MISS) { for(i=1; icaches->nelts; i++) { subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - if(subcache->tile_get(ctx, subcache, tile) == MAPCACHE_SUCCESS) { + if(mapcache_cache_tile_get(ctx, subcache, tile) == MAPCACHE_SUCCESS) { ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from secondary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); for(--i;i>=0;i--) { subcache = APR_ARRAY_IDX(cache->caches,i,mapcache_cache*); - subcache->tile_set(ctx, subcache, tile); + mapcache_cache_tile_set(ctx, subcache, tile); ctx->clear_errors(ctx); /* silently ignore these errors */ ctx->log(ctx,MAPCACHE_DEBUG,"transferring tile (%s,z=%d,y=%d,x=%d) to cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); } @@ -83,7 +91,7 @@ } return MAPCACHE_CACHE_MISS; } else { - ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from primary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); + //ctx->log(ctx,MAPCACHE_DEBUG,"got tile (%s,z=%d,y=%d,x=%d) from primary cache (%s)",tile->tileset->name, tile->z, tile->y, tile->x, subcache->name); return ret; } } @@ -92,22 +100,14 @@ { mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,cache->caches->nelts-1,mapcache_cache*); - return subcache->tile_set(ctx, subcache, tile); + return mapcache_cache_tile_set(ctx, subcache, tile); } static void _mapcache_cache_multitier_tile_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { mapcache_cache_multitier *cache = (mapcache_cache_multitier*)pcache; mapcache_cache *subcache = APR_ARRAY_IDX(cache->caches,cache->caches->nelts-1,mapcache_cache*); - if(subcache->tile_multi_set) { - return subcache->tile_multi_set(ctx, subcache, tiles, ntiles); - } else { - int i; - for( i=0;itile_set(ctx, subcache, &tiles[i]); - GC_CHECK_ERROR(ctx); - } - } + return mapcache_cache_tile_multi_set(ctx, subcache, tiles, ntiles); } /** @@ -153,11 +153,11 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_COMPOSITE; - cache->cache.tile_delete = _mapcache_cache_multitier_tile_delete; - cache->cache.tile_get = _mapcache_cache_multitier_tile_get; - cache->cache.tile_exists = _mapcache_cache_multitier_tile_exists; - cache->cache.tile_set = _mapcache_cache_multitier_tile_set; - cache->cache.tile_multi_set = _mapcache_cache_multitier_tile_multi_set; + cache->cache._tile_delete = _mapcache_cache_multitier_tile_delete; + cache->cache._tile_get = _mapcache_cache_multitier_tile_get; + cache->cache._tile_exists = _mapcache_cache_multitier_tile_exists; + cache->cache._tile_set = _mapcache_cache_multitier_tile_set; + cache->cache._tile_multi_set = _mapcache_cache_multitier_tile_multi_set; cache->cache.configuration_post_config = _mapcache_cache_multitier_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_multitier_configuration_parse_xml; return (mapcache_cache*)cache; diff -Nru mapcache-1.4.0/lib/cache_rest.c mapcache-1.6.1/lib/cache_rest.c --- mapcache-1.4.0/lib/cache_rest.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_rest.c 2017-09-29 21:31:13.000000000 +0000 @@ -35,12 +35,139 @@ #include #include #include +#include +#include + +typedef struct mapcache_cache_rest mapcache_cache_rest; +typedef struct mapcache_cache_s3 mapcache_cache_s3; +typedef struct mapcache_cache_azure mapcache_cache_azure; +typedef struct mapcache_cache_google mapcache_cache_google; + +typedef enum { + MAPCACHE_REST_METHOD_GET, + MAPCACHE_REST_METHOD_HEAD, + MAPCACHE_REST_METHOD_PUT, + MAPCACHE_REST_METHOD_POST, + MAPCACHE_REST_METHOD_DELETE +} mapcache_rest_method; + +typedef enum { + MAPCACHE_REST_PROVIDER_NONE, + MAPCACHE_REST_PROVIDER_S3, + MAPCACHE_REST_PROVIDER_AZURE, + MAPCACHE_REST_PROVIDER_GOOGLE +} mapcache_rest_provider; + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest); +void hmac_sha256(const unsigned char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + unsigned char *mac, unsigned mac_size); +void hmac_sha1(const char *message, unsigned int message_len, + const unsigned char *key, unsigned int key_size, + void *mac); +void sha_hex_encode(unsigned char *sha, unsigned int sha_size); +char *base64_encode(apr_pool_t *pool, const unsigned char *data, size_t input_length); + +typedef struct mapcache_rest_operation mapcache_rest_operation; +struct mapcache_rest_operation { + apr_table_t *headers; + mapcache_rest_method method; + char *tile_url; + char *header_file; + void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); +}; + +typedef struct mapcache_rest_configuration mapcache_rest_configuration; +struct mapcache_rest_configuration { + apr_table_t *common_headers; + char *tile_url; + char *header_file; + mapcache_rest_operation has_tile; + mapcache_rest_operation get_tile; + mapcache_rest_operation set_tile; + mapcache_rest_operation multi_set_tile; + mapcache_rest_operation delete_tile; + void (*add_headers)(mapcache_context *ctx, mapcache_cache_rest *pcache, mapcache_tile *tile, char *url, apr_table_t *headers); +}; + +/**\class mapcache_cache_rest + * \brief a mapcache_cache on a 3rd party HTTP Rest API + * \implements mapcache_cache + */ +struct mapcache_cache_rest { + mapcache_cache cache; + mapcache_rest_configuration rest; + int use_redirects; + int timeout; + int connection_timeout; + int detect_blank; + mapcache_rest_provider provider; +}; + +struct mapcache_cache_s3 { + mapcache_cache_rest cache; + char *id; + char *secret; + char *region; + char *credentials_file; +}; + +struct mapcache_cache_azure { + mapcache_cache_rest cache; + char *id; + char *secret; + char *container; +}; + +struct mapcache_cache_google { + mapcache_cache_rest cache; + char *access; + char *secret; +}; typedef struct { mapcache_buffer *buffer; size_t offset; } buffer_struct; +struct rest_conn_params { + mapcache_cache_rest *cache; +}; + +void mapcache_rest_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + CURL *curl_handle = curl_easy_init(); + if(!curl_handle) { + ctx->set_error(ctx,500,"failed to create curl handle"); + *conn_ = NULL; + return; + } + *conn_ = curl_handle; +} + +void mapcache_rest_connection_destructor(void *conn_) { + CURL *curl_handle = (CURL*) conn_; + curl_easy_cleanup(curl_handle); +} + +static mapcache_pooled_connection* _rest_get_connection(mapcache_context *ctx, mapcache_cache_rest *cache, mapcache_tile *tile) +{ + mapcache_pooled_connection *pc; + struct rest_conn_params params; + + params.cache = cache; + + pc = mapcache_connection_pool_get_connection(ctx,cache->cache.name,mapcache_rest_connection_constructor, + mapcache_rest_connection_destructor, ¶ms); + if(!GC_HAS_ERROR(ctx) && pc && pc->connection) { + CURL *curl_handle = (CURL*)pc->connection; + curl_easy_reset(curl_handle); + curl_easy_setopt(curl_handle, CURLOPT_CONNECTTIMEOUT, cache->connection_timeout); + curl_easy_setopt(curl_handle, CURLOPT_TIMEOUT, cache->timeout); + } + + return pc; +} + static size_t buffer_read_callback(void *ptr, size_t size, size_t nmemb, void *stream) { buffer_struct *buffer = (buffer_struct*)stream; @@ -69,7 +196,11 @@ apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; int i; for (i = 0; i < array->nelts; i++) { - curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",elts[i].val,NULL)); + if(strlen(elts[i].val) > 0) { + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",elts[i].val,NULL)); + } else { + curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,":",NULL)); + } } curl_easy_setopt(curl, CURLOPT_HTTPHEADER, curl_headers); } @@ -86,46 +217,49 @@ response = mapcache_buffer_create(10,ctx->pool); #if LIBCURL_VERSION_NUM < 0x071700 - /* + /* * hack around a bug in curl <= 7.22 where the content-length is added * a second time even if ti was present in the manually set headers */ apr_table_unset(headers, "Content-Length"); #endif - _set_headers(ctx, curl, headers); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); - /* we want to use our own read function */ + /* we want to use our own read function */ curl_easy_setopt(curl, CURLOPT_READFUNCTION, buffer_read_callback); - /* enable uploading */ + /* enable uploading */ curl_easy_setopt(curl, CURLOPT_UPLOAD, 1L); - /* HTTP PUT please */ + /* HTTP PUT please */ curl_easy_setopt(curl, CURLOPT_PUT, 1L); + /* don't use an Expect: 100 Continue header */ + apr_table_set(headers, "Expect", ""); + _set_headers(ctx, curl, headers); + /* specify target URL, and note that this URL should include a file - * name, not only a directory */ + * name, not only a directory */ curl_easy_setopt(curl, CURLOPT_URL, url); - /* now specify which file to upload */ + /* now specify which file to upload */ curl_easy_setopt(curl, CURLOPT_READDATA, &data); /* provide the size of the upload, we specicially typecast the value - * to curl_off_t since we must be sure to use the correct data size */ + * to curl_off_t since we must be sure to use the correct data size */ curl_easy_setopt(curl, CURLOPT_INFILESIZE, buffer->size); - + /* send all data to this function */ curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, buffer_write_callback); /* we pass our mapcache_buffer struct to the callback function */ curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)response); - /* Now run off and do what you've been told! */ + /* Now run off and do what you've been told! */ res = curl_easy_perform(curl); - /* Check for errors */ + /* Check for errors */ if(res != CURLE_OK) { ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest put: %s",curl_easy_strerror(res)); } else { @@ -140,32 +274,24 @@ } -static int _head_request(mapcache_context *ctx, char *url, apr_table_t *headers) { +static int _head_request(mapcache_context *ctx, CURL *curl, char *url, apr_table_t *headers) { - CURL *curl; CURLcode res; long http_code; - - curl = curl_easy_init(); - - if(!curl) { - ctx->set_error(ctx,500,"failed to create curl handle"); - return -1; - } - - _set_headers(ctx, curl, headers); + _set_headers(ctx, curl, headers); + curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); /* specify target URL, and note that this URL should include a file - * name, not only a directory */ + * name, not only a directory */ curl_easy_setopt(curl, CURLOPT_URL, url); - + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); - /* Now run off and do what you've been told! */ + /* Now run off and do what you've been told! */ res = curl_easy_perform(curl); - /* Check for errors */ + /* Check for errors */ if(res != CURLE_OK) { ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest head %s",curl_easy_strerror(res)); http_code = 500; @@ -173,63 +299,44 @@ curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); } - /* always cleanup */ - curl_easy_cleanup(curl); - return (int)http_code; } -static int _delete_request(mapcache_context *ctx, char *url, apr_table_t *headers) { +static int _delete_request(mapcache_context *ctx, CURL *curl, char *url, apr_table_t *headers) { - CURL *curl; CURLcode res; long http_code; - curl = curl_easy_init(); - if(!curl) { - ctx->set_error(ctx,500,"failed to create curl handle"); - return -1; - } - _set_headers(ctx, curl, headers); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); /* specify target URL, and note that this URL should include a file - * name, not only a directory */ + * name, not only a directory */ curl_easy_setopt(curl, CURLOPT_URL, url); - + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "DELETE"); + curl_easy_setopt(curl, CURLOPT_NOBODY, 1); - /* Now run off and do what you've been told! */ + /* Now run off and do what you've been told! */ res = curl_easy_perform(curl); - /* Check for errors */ + /* Check for errors */ if(res != CURLE_OK) { - ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest head %s",curl_easy_strerror(res)); + ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest delete %s",curl_easy_strerror(res)); http_code = 500; } else { curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); } - /* always cleanup */ - curl_easy_cleanup(curl); - return (int)http_code; } -static mapcache_buffer* _get_request(mapcache_context *ctx, char *url, apr_table_t *headers) { +static mapcache_buffer* _get_request(mapcache_context *ctx, CURL *curl, char *url, apr_table_t *headers) { - CURL *curl; CURLcode res; mapcache_buffer *data = NULL; long http_code; - curl = curl_easy_init(); - if(!curl) { - ctx->set_error(ctx,500,"failed to create curl handle"); - return NULL; - } - _set_headers(ctx, curl, headers); curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1); @@ -243,18 +350,18 @@ curl_easy_setopt(curl, CURLOPT_WRITEDATA, (void *)data); /* specify target URL, and note that this URL should include a file - * name, not only a directory */ + * name, not only a directory */ curl_easy_setopt(curl, CURLOPT_URL, url); - /* Now run off and do what you've been told! */ + /* Now run off and do what you've been told! */ res = curl_easy_perform(curl); - /* Check for errors */ + /* Check for errors */ if(res != CURLE_OK) { ctx->set_error(ctx, 500, "curl_easy_perform() failed in rest get: %s",curl_easy_strerror(res)); data = NULL; } else { curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code); - /* handle special behavior of s3 */ + /* handle special behavior of s3 */ if(http_code == 403) { char *msg = data->buf; while(msg && *msg) { @@ -277,33 +384,102 @@ } } - /* always cleanup */ - curl_easy_cleanup(curl); - return data; } +/** + * @brief _mapcache_cache_rest_add_headers_from_file populate header table from entries found in file + * @param ctx + * @param file the file from which headers should be read + * @param headers the output table which will be populated + */ +void _mapcache_cache_rest_add_headers_from_file(mapcache_context *ctx, char *file, apr_table_t *headers) { + apr_status_t rv; + apr_file_t *f; + if((rv=apr_file_open(&f, file, APR_FOPEN_READ|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,APR_OS_DEFAULT, + ctx->pool)) == APR_SUCCESS) { + char line[8096]; + while( (rv = apr_file_gets(line,8096,f))== APR_SUCCESS) { + char *header_name=line, *header_val=line, *header_endval; + int found_token = MAPCACHE_FALSE; + /*search for header delimiter (:)*/ + while(header_val && *header_val) { + if(*header_val == ':') { + *header_val = '\0'; + found_token = MAPCACHE_TRUE; + break; + } + header_val++; + } + if(!found_token) { + /* malformed line, silently skip it */ + continue; + } + + header_val++; + + if(!*header_val) { + /* malformed/empty line, skip it */ + continue; + } + + header_endval = header_val; + while(*header_endval) { + if(*header_endval == '\r' || *header_endval == '\n') { + *header_endval = '\0'; + break; + } + header_endval++; + } + + if(!*header_val) { + /* empty header value, skip it */ + continue; + } + + apr_table_set(headers, header_name, header_val); + } + apr_file_close(f); + } else { + ctx->set_error(ctx,500,"rest cache: failed to access header file"); + } +} + apr_table_t* _mapcache_cache_rest_headers(mapcache_context *ctx, mapcache_tile *tile, mapcache_rest_configuration *config, mapcache_rest_operation *operation) { apr_table_t *ret = apr_table_make(ctx->pool,3); const apr_array_header_t *array; - + if(config->common_headers) { - array = apr_table_elts(config->common_headers); - apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; + apr_table_entry_t *elts; int i; + array = apr_table_elts(config->common_headers); + elts = (apr_table_entry_t *) array->elts; for (i = 0; i < array->nelts; i++) { apr_table_set(ret, elts[i].key,elts[i].val); } } + if(config->header_file) { + _mapcache_cache_rest_add_headers_from_file(ctx,config->header_file,ret); + if(GC_HAS_ERROR(ctx)) { + return NULL; + } + } if(operation->headers) { - array = apr_table_elts(operation->headers); - apr_table_entry_t *elts = (apr_table_entry_t *) array->elts; + apr_table_entry_t *elts; int i; + array = apr_table_elts(operation->headers); + elts = (apr_table_entry_t *) array->elts; for (i = 0; i < array->nelts; i++) { apr_table_set(ret, elts[i].key,elts[i].val); } } + if(operation->header_file) { + _mapcache_cache_rest_add_headers_from_file(ctx,operation->header_file,ret); + if(GC_HAS_ERROR(ctx)) { + return NULL; + } + } return ret; } @@ -374,14 +550,25 @@ apr_psprintf(ctx->pool,"%d", tile->grid_link->grid->nlevels - tile->z - 1)); if(tile->dimensions) { - char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; - while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->key,"#",entry->val,NULL); + int i; + if(strstr(*url,"{dim")) { + char *dimstring=""; + i = tile->dimensions->nelts; + while(i--) { + char *single_dim; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + if(!entry->cached_value) { + ctx->set_error(ctx,500,"BUG: dimension (%s) not defined",entry->dimension->name); + return; + } + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",entry->cached_value,NULL); + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(*url,single_dim)) { + *url = mapcache_util_str_replace(ctx->pool,*url, single_dim, entry->cached_value); + } + } + *url = mapcache_util_str_replace(ctx->pool,*url, "{dim}", dimstring); } - *url = mapcache_util_str_replace(ctx->pool,*url, "{dim}", dimstring); } /* url-encode everything after the host name */ @@ -402,7 +589,7 @@ *url = apr_pstrcat(ctx->pool,*url,path,NULL); - /*ctx->log(ctx,MAPCACHE_WARN,"rest url: %s",*url);*/ + /*ctx->log(ctx,MAPCACHE_DEBUG,"rest url: %s",*url);*/ } @@ -453,19 +640,29 @@ } } +static const char* my_apr_table_get(apr_table_t *t, char *key) { + const char *val = apr_table_get(t,key); + if(!val) val = ""; + return val; +} + + static void _mapcache_cache_google_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) { char *stringToSign, **aheaders, *resource = url, x_amz_date[64]; const char *head; const apr_array_header_t *ahead; apr_table_entry_t *elts; - int i,nCanonicalHeaders=0,cnt=0; - assert(rcache->provider == MAPCACHE_REST_PROVIDER_GOOGLE); - mapcache_cache_google *google = (mapcache_cache_google*)rcache; - time_t now = time(NULL); - struct tm *tnow = gmtime(&now); + mapcache_cache_google *google; + time_t now; + struct tm *tnow; unsigned char sha[65]; char b64[150]; + int i,nCanonicalHeaders=0,cnt=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_GOOGLE); + google = (mapcache_cache_google*)rcache; + now = time(NULL); + tnow = gmtime(&now); sha[64]=0; strftime(x_amz_date, 64 , "%a, %d %b %Y %H:%M:%S GMT", tnow); @@ -478,12 +675,11 @@ apr_table_set(headers, "Content-MD5", b64); } - head = apr_table_get(headers, "Content-MD5"); - if(!head) head = ""; + head = my_apr_table_get(headers, "Content-MD5"); stringToSign=apr_pstrcat(ctx->pool, method, "\n", head, "\n", NULL); - head = apr_table_get(headers, "Content-Type"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - + head = my_apr_table_get(headers, "Content-Type"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + /* Date: header, left empty as we are using x-amz-date */ stringToSign=apr_pstrcat(ctx->pool, stringToSign, "\n", NULL); @@ -506,7 +702,7 @@ for(i=0; ipool, stringToSign, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); } - + /* find occurence of third "/" in url */ while(*resource) { if(*resource == '/') cnt++; @@ -527,19 +723,23 @@ apr_table_set( headers, "Authorization", apr_pstrcat(ctx->pool,"AWS ", google->access, ":", b64, NULL)); } + static void _mapcache_cache_azure_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) { char *stringToSign, **aheaders, *canonical_headers="", *canonical_resource=NULL, *resource = url, x_ms_date[64]; const char *head; const apr_array_header_t *ahead; apr_table_entry_t *elts; - int i,nCanonicalHeaders=0,cnt=0; - assert(rcache->provider == MAPCACHE_REST_PROVIDER_AZURE); - mapcache_cache_azure *azure = (mapcache_cache_azure*)rcache; - time_t now = time(NULL); - struct tm *tnow = gmtime(&now); + mapcache_cache_azure *azure; + time_t now; + struct tm *tnow; unsigned char sha[65]; char *b64sign,*keyub64; + int i,nCanonicalHeaders=0,cnt=0; + assert(rcache->provider == MAPCACHE_REST_PROVIDER_AZURE); + azure = (mapcache_cache_azure*)rcache; + now = time(NULL); + tnow = gmtime(&now); sha[64]=0; strftime(x_ms_date, sizeof(x_ms_date), "%a, %d %b %Y %H:%M:%S GMT", tnow); @@ -548,36 +748,37 @@ apr_table_set(headers,"x-ms-blob-type","BlockBlob"); stringToSign = apr_pstrcat(ctx->pool, method, "\n", NULL); - head = apr_table_get(headers, "Content-Encoding"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "Content-Language"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "Content-Length"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "Content-MD5"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "Content-Type"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "Date"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "If-Modified-Since"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "If-Match"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "If-None-Match"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "If-Unmodified-Since"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); - head = apr_table_get(headers, "Range"); - if(!head) head = ""; stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Encoding"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Language"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Length"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-MD5"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Content-Type"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Date"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-Modified-Since"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-Match"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-None-Match"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "If-Unmodified-Since"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); + head = my_apr_table_get(headers, "Range"); + stringToSign=apr_pstrcat(ctx->pool, stringToSign, head, "\n", NULL); ahead = apr_table_elts(headers); aheaders = apr_pcalloc(ctx->pool, ahead->nelts * sizeof(char*)); elts = (apr_table_entry_t *) ahead->elts; for (i = 0; i < ahead->nelts; i++) { + char *k; if(strncmp(elts[i].key,"x-ms-",5) || elts[i].key[5]==0) continue; - char *k = aheaders[nCanonicalHeaders] = apr_pstrdup(ctx->pool, elts[i].key); + k = aheaders[nCanonicalHeaders] = apr_pstrdup(ctx->pool, elts[i].key); while(*k) { *k = tolower(*k); k++; @@ -589,7 +790,7 @@ for(i=0; ipool, canonical_headers, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); } - + /* find occurence of third "/" in url */ while(*resource) { if(*resource == '/') cnt++; @@ -616,8 +817,23 @@ apr_table_set( headers, "Authorization", apr_pstrcat(ctx->pool,"SharedKey ", azure->id, ":", b64sign, NULL)); - + } + +static void _remove_lineends(char *str) { + if(str) { + size_t len = strlen(str); + while(len>0) { + if(str[len-1] == '\n' || str[len-1] == '\r') { + str[len-1] = '\0'; + len--; + } else { + break; + } + } + } +} + static void _mapcache_cache_s3_headers_add(mapcache_context *ctx, const char* method, mapcache_cache_rest *rcache, mapcache_tile *tile, char *url, apr_table_t *headers) { unsigned char sha1[65],sha2[65]; @@ -627,15 +843,52 @@ const apr_array_header_t *ahead; char *tosign, *key, *canonical_request, x_amz_date[64], *resource = url, **aheaders, *auth; apr_table_entry_t *elts; + mapcache_cache_s3 *s3; + char *aws_access_key_id = NULL, *aws_secret_access_key = NULL, *aws_security_token = NULL; sha1[64]=sha2[64]=0; assert(rcache->provider == MAPCACHE_REST_PROVIDER_S3); - mapcache_cache_s3 *s3 = (mapcache_cache_s3*)rcache; + s3 = (mapcache_cache_s3*)rcache; + + if(s3->credentials_file) { + apr_status_t rv; + apr_file_t *f; + if((rv=apr_file_open(&f, s3->credentials_file, + APR_FOPEN_READ|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,APR_OS_DEFAULT, + ctx->pool)) == APR_SUCCESS) { + char line[1024]; + if( (rv = apr_file_gets(line,1024,f))== APR_SUCCESS) { + _remove_lineends(line); + aws_access_key_id = apr_pstrdup(ctx->pool,line); + } + if( (rv = apr_file_gets(line,1024,f))== APR_SUCCESS) { + _remove_lineends(line); + aws_secret_access_key = apr_pstrdup(ctx->pool,line); + } + if( (rv = apr_file_gets(line,1024,f))== APR_SUCCESS) { + _remove_lineends(line); + aws_security_token = apr_pstrdup(ctx->pool,line); + } + apr_file_close(f); + if(!aws_access_key_id || !*aws_access_key_id|| !aws_secret_access_key ||!*aws_secret_access_key) { + ctx->set_error(ctx,500,"failed to read access or secret key from credentials file"); + } + if(aws_security_token && !*aws_security_token) { + aws_security_token = NULL; + } + } else { + ctx->set_error(ctx,500,"failed to access S3 credential config"); + } + } else { + aws_access_key_id = s3->id; + aws_secret_access_key = s3->secret; + aws_security_token = NULL; + } if(!strcmp(method,"PUT")) { assert(tile->encoded_data); sha256((unsigned char*)tile->encoded_data->buf, tile->encoded_data->size, sha1); - sha_hex_encode(sha1,32); + sha_hex_encode(sha1,32); } else { /* sha256 hash of empty string */ memcpy(sha1,"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",64); @@ -653,10 +906,14 @@ ctx->set_error(ctx,500,"invalid s3 url provided"); return; } - + strftime(x_amz_date, sizeof(x_amz_date), "%Y%m%dT%H%M%SZ", tnow); apr_table_set(headers, "x-amz-date", x_amz_date); + if(aws_security_token) { + apr_table_set(headers, "x-amz-security-token", aws_security_token); + } + canonical_request = apr_pstrcat(ctx->pool, method, "\n" ,resource, "\n\n",NULL); ahead = apr_table_elts(headers); @@ -670,7 +927,7 @@ k++; } } - + header_gnome_sort(aheaders, ahead->nelts); for(i=0; inelts; i++) { canonical_request = apr_pstrcat(ctx->pool, canonical_request, aheaders[i],":",apr_table_get(headers,aheaders[i]),"\n",NULL); @@ -693,7 +950,7 @@ tosign = apr_pstrcat(ctx->pool, tosign, x_amz_date, "/", s3->region, "/s3/aws4_request\n", sha1,NULL); //printf("key to sign: %s\n",tosign); - key = apr_pstrcat(ctx->pool, "AWS4", s3->secret, NULL); + key = apr_pstrcat(ctx->pool, "AWS4", aws_secret_access_key, NULL); hmac_sha256((unsigned char*)x_amz_date, 8, (unsigned char*)key, strlen(key), sha1, 32); hmac_sha256((unsigned char*)s3->region, strlen(s3->region), sha1, 32, sha2, 32); hmac_sha256((unsigned char*)"s3", 2, sha2, 32, sha1, 32); @@ -702,7 +959,7 @@ sha_hex_encode(sha1,32); - auth = apr_pstrcat(ctx->pool, "AWS4-HMAC-SHA256 Credential=",s3->id,"/",x_amz_date,"/",s3->region,"/s3/aws4_request,SignedHeaders=",NULL); + auth = apr_pstrcat(ctx->pool, "AWS4-HMAC-SHA256 Credential=",aws_access_key_id,"/",x_amz_date,"/",s3->region,"/s3/aws4_request,SignedHeaders=",NULL); for(i=0; inelts; i++) { if(i==ahead->nelts-1) { @@ -760,20 +1017,38 @@ char *url; apr_table_t *headers; int status; + mapcache_pooled_connection *pc; + CURL *curl; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.has_tile, &url); headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.has_tile); + + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + if(rcache->rest.add_headers) { rcache->rest.add_headers(ctx,rcache,tile,url,headers); } if(rcache->rest.has_tile.add_headers) { rcache->rest.has_tile.add_headers(ctx,rcache,tile,url,headers); } - - status = _head_request(ctx, url, headers); + pc = _rest_get_connection(ctx, rcache, tile); if(GC_HAS_ERROR(ctx)) return MAPCACHE_FAILURE; + curl = pc->connection; + + status = _head_request(ctx, curl, url, headers); + + + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return MAPCACHE_FAILURE; + } + + mapcache_connection_pool_release_connection(ctx,pc); + if( status == 200) return MAPCACHE_TRUE; else @@ -786,19 +1061,32 @@ char *url; apr_table_t *headers; int status; + mapcache_pooled_connection *pc; + CURL *curl; _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.delete_tile, &url); headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.delete_tile); + GC_CHECK_ERROR(ctx); + if(rcache->rest.add_headers) { rcache->rest.add_headers(ctx,rcache,tile,url,headers); } if(rcache->rest.delete_tile.add_headers) { rcache->rest.delete_tile.add_headers(ctx,rcache,tile,url,headers); } - - status = _delete_request(ctx, url, headers); + + pc = _rest_get_connection(ctx, rcache, tile); GC_CHECK_ERROR(ctx); - if(status!=200 && status!=202 && status!=204) { + curl = pc->connection; + + status = _delete_request(ctx, curl, url, headers); + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return; + } + mapcache_connection_pool_release_connection(ctx,pc); + + if(status!=200 && status!=202 && status!=204 && status!=404 && status!=410) { //ctx->set_error(ctx,500,"rest delete returned code %d", status); } } @@ -816,98 +1104,110 @@ mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; char *url; apr_table_t *headers; + mapcache_pooled_connection *pc; + CURL *curl; _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.get_tile, &url); if(tile->allow_redirect && rcache->use_redirects) { tile->redirect = url; return MAPCACHE_SUCCESS; } headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.get_tile); + + if(GC_HAS_ERROR(ctx)) + return MAPCACHE_FAILURE; + if(rcache->rest.add_headers) { rcache->rest.add_headers(ctx,rcache,tile,url,headers); } if(rcache->rest.get_tile.add_headers) { rcache->rest.get_tile.add_headers(ctx,rcache,tile,url,headers); } - tile->encoded_data = _get_request(ctx, url, headers); + + pc = _rest_get_connection(ctx, rcache, tile); if(GC_HAS_ERROR(ctx)) return MAPCACHE_FAILURE; + curl = pc->connection; + + tile->encoded_data = _get_request(ctx, curl, url, headers); + + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return MAPCACHE_FAILURE; + } + mapcache_connection_pool_release_connection(ctx,pc); + if(!tile->encoded_data) return MAPCACHE_CACHE_MISS; return MAPCACHE_SUCCESS; } -static void _mapcache_cache_rest_multi_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tiles, int ntiles) { +static void _mapcache_cache_rest_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { mapcache_cache_rest *rcache = (mapcache_cache_rest*)pcache; char *url; apr_table_t *headers; - CURL *curl = curl_easy_init(); - int i; + mapcache_pooled_connection *pc; + CURL *curl; - if(!curl) { - ctx->set_error(ctx,500,"failed to create curl handle"); - return; - } - for(i=0; irest, &rcache->rest.set_tile, &url); - headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.set_tile); - - if(!tile->encoded_data) { - tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); - if(GC_HAS_ERROR(ctx)) { - goto multi_put_cleanup; - } + if(rcache->detect_blank) { + if(tile->nodata) { + return; } - - apr_table_set(headers,"Content-Length",apr_psprintf(ctx->pool,"%lu",tile->encoded_data->size)); - if(tile->tileset->format && tile->tileset->format->mime_type) - apr_table_set(headers, "Content-Type", tile->tileset->format->mime_type); - else { - mapcache_image_format_type imgfmt = mapcache_imageio_header_sniff(ctx,tile->encoded_data); - if(imgfmt == GC_JPEG) { - apr_table_set(headers, "Content-Type", "image/jpeg"); - } else if (imgfmt == GC_PNG) { - apr_table_set(headers, "Content-Type", "image/png"); + if(!tile->raw_image) { + tile->raw_image = mapcache_imageio_decode(ctx, tile->encoded_data); + GC_CHECK_ERROR(ctx); + } + if(mapcache_image_blank_color(tile->raw_image) != MAPCACHE_FALSE) { + if(tile->raw_image->data[3] == 0) { + /* We have a blank (uniform) image who's first pixel is fully transparent, thus the whole image is transparent */ + tile->nodata = 1; + return; } } + } - if(rcache->rest.add_headers) { - rcache->rest.add_headers(ctx,rcache,tile,url,headers); - } - if(rcache->rest.set_tile.add_headers) { - rcache->rest.set_tile.add_headers(ctx,rcache,tile,url,headers); - } - _put_request(ctx, curl, tile->encoded_data, url, headers); - if(GC_HAS_ERROR(ctx)) { - goto multi_put_cleanup; + _mapcache_cache_rest_tile_url(ctx, tile, &rcache->rest, &rcache->rest.set_tile, &url); + headers = _mapcache_cache_rest_headers(ctx, tile, &rcache->rest, &rcache->rest.set_tile); + GC_CHECK_ERROR(ctx); + + if(!tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + + apr_table_set(headers,"Content-Length",apr_psprintf(ctx->pool,"%lu",tile->encoded_data->size)); + if(tile->tileset->format && tile->tileset->format->mime_type) + apr_table_set(headers, "Content-Type", tile->tileset->format->mime_type); + else { + mapcache_image_format_type imgfmt = mapcache_imageio_header_sniff(ctx,tile->encoded_data); + if(imgfmt == GC_JPEG) { + apr_table_set(headers, "Content-Type", "image/jpeg"); + } else if (imgfmt == GC_PNG) { + apr_table_set(headers, "Content-Type", "image/png"); } } -multi_put_cleanup: - /* always cleanup */ - curl_easy_cleanup(curl); + if(rcache->rest.add_headers) { + rcache->rest.add_headers(ctx,rcache,tile,url,headers); + } + if(rcache->rest.set_tile.add_headers) { + rcache->rest.set_tile.add_headers(ctx,rcache,tile,url,headers); + } -} + pc = _rest_get_connection(ctx, rcache, tile); + GC_CHECK_ERROR(ctx); + curl = pc->connection; -/** - * \brief write tile data to rest backend - * - * writes the content of mapcache_tile::data to disk. - * \returns MAPCACHE_FAILURE if there is no data to write, or if the tile isn't locked - * \returns MAPCACHE_SUCCESS if the tile has been successfully written to disk - * \private \memberof mapcache_cache_rest - * \sa mapcache_cache::tile_set() - */ -static void _mapcache_cache_rest_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) -{ - return _mapcache_cache_rest_multi_set(ctx, pcache, tile, 1); -} + _put_request(ctx, curl, tile->encoded_data, url, headers); + if(GC_HAS_ERROR(ctx)) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + return; + } + mapcache_connection_pool_release_connection(ctx,pc); +} static void _mapcache_cache_rest_operation_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_rest_operation *op) { @@ -919,6 +1219,9 @@ apr_table_set(op->headers, header_node->name, header_node->txt); } } + if ((cur_node = ezxml_child(node,"header_file")) != NULL) { + op->header_file = apr_pstrdup(ctx->pool, cur_node->txt); + } } /** @@ -936,6 +1239,37 @@ dcache->use_redirects = 1; } } + if ((cur_node = ezxml_child(node,"connection_timeout")) != NULL) { + char *endptr; + dcache->connection_timeout = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0 || dcache->connection_timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache \"%s\" (positive integer expected)", + cur_node->txt); + return; + } + } else { + dcache->connection_timeout = 30; + } + + if ((cur_node = ezxml_child(node,"timeout")) != NULL) { + char *endptr; + dcache->timeout = (int)strtol(cur_node->txt,&endptr,10); + if(*endptr != 0 || dcache->timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache \"%s\" (positive integer expected)", + cur_node->txt); + return; + } + } else { + dcache->timeout = 120; + } + + dcache->detect_blank = 0; + if ((cur_node = ezxml_child(node, "detect_blank")) != NULL) { + if(strcasecmp(cur_node->txt,"false")) { + dcache->detect_blank = 1; + } + } + if ((cur_node = ezxml_child(node,"headers")) != NULL) { ezxml_t header_node; dcache->rest.common_headers = apr_table_make(ctx->pool,3); @@ -943,6 +1277,12 @@ apr_table_set(dcache->rest.common_headers, header_node->name, header_node->txt); } } + + if ((cur_node = ezxml_child(node,"header_file")) != NULL) { + dcache->rest.header_file = apr_pstrdup(ctx->pool, cur_node->txt); + } + + for(cur_node = ezxml_child(node,"operation"); cur_node; cur_node = cur_node->next) { char *type = (char*)ezxml_attr(cur_node,"type"); if(!type) { @@ -969,7 +1309,7 @@ } -static void _mapcache_cache_google_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_google_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { ezxml_t cur_node; mapcache_cache_google *google = (mapcache_cache_google*)cache; @@ -989,23 +1329,31 @@ } } -static void _mapcache_cache_s3_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_s3_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { ezxml_t cur_node; mapcache_cache_s3 *s3 = (mapcache_cache_s3*)cache; _mapcache_cache_rest_configuration_parse_xml(ctx, node, cache, config); GC_CHECK_ERROR(ctx); - if ((cur_node = ezxml_child(node,"id")) != NULL) { - s3->id = apr_pstrdup(ctx->pool, cur_node->txt); - } else { - ctx->set_error(ctx,400,"s3 cache (%s) is missing required child", cache->name); - return; - } - if ((cur_node = ezxml_child(node,"secret")) != NULL) { - s3->secret = apr_pstrdup(ctx->pool, cur_node->txt); + if ((cur_node = ezxml_child(node,"credentials_file")) != NULL) { + s3->credentials_file = apr_pstrdup(ctx->pool, cur_node->txt); } else { - ctx->set_error(ctx,400,"s3 cache (%s) is missing required child", cache->name); - return; + if ((cur_node = ezxml_child(node,"id")) != NULL) { + s3->id = apr_pstrdup(ctx->pool, cur_node->txt); + } else if ( getenv("AWS_ACCESS_KEY_ID")) { + s3->id = apr_pstrdup(ctx->pool,getenv("AWS_ACCESS_KEY_ID")); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child or AWS_ACCESS_KEY_ID environment", cache->name); + return; + } + if ((cur_node = ezxml_child(node,"secret")) != NULL) { + s3->secret = apr_pstrdup(ctx->pool, cur_node->txt); + } else if ( getenv("AWS_SECRET_ACCESS_KEY")) { + s3->secret = apr_pstrdup(ctx->pool,getenv("AWS_SECRET_ACCESS_KEY")); + } else { + ctx->set_error(ctx,400,"s3 cache (%s) is missing required child or AWS_SECRET_ACCESS_KEY environment", cache->name); + return; + } } if ((cur_node = ezxml_child(node,"region")) != NULL) { s3->region = apr_pstrdup(ctx->pool, cur_node->txt); @@ -1015,7 +1363,7 @@ } } -static void _mapcache_cache_azure_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) +static void _mapcache_cache_azure_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { ezxml_t cur_node; mapcache_cache_azure *azure = (mapcache_cache_azure*)cache; @@ -1067,7 +1415,6 @@ } void mapcache_cache_rest_init(mapcache_context *ctx, mapcache_cache_rest *cache) { - cache->retry_count = 0; cache->use_redirects = 0; cache->rest.get_tile.method = MAPCACHE_REST_METHOD_GET; cache->rest.set_tile.method = MAPCACHE_REST_METHOD_PUT; @@ -1076,11 +1423,10 @@ cache->rest.has_tile.method = MAPCACHE_REST_METHOD_HEAD; cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_REST; - cache->cache.tile_delete = _mapcache_cache_rest_delete; - cache->cache.tile_get = _mapcache_cache_rest_get; - cache->cache.tile_exists = _mapcache_cache_rest_has_tile; - cache->cache.tile_set = _mapcache_cache_rest_set; - cache->cache.tile_multi_set = _mapcache_cache_rest_multi_set; + cache->cache._tile_delete = _mapcache_cache_rest_delete; + cache->cache._tile_get = _mapcache_cache_rest_get; + cache->cache._tile_exists = _mapcache_cache_rest_has_tile; + cache->cache._tile_set = _mapcache_cache_rest_set; cache->cache.configuration_post_config = _mapcache_cache_rest_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_rest_configuration_parse_xml; } diff -Nru mapcache-1.4.0/lib/cache_riak.c mapcache-1.6.1/lib/cache_riak.c --- mapcache-1.4.0/lib/cache_riak.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_riak.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,10 +27,10 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_RIAK -#include "mapcache.h" +#include #include #include @@ -39,17 +39,25 @@ #include #include -/* - * Since we don't construct the connection pool and store it in the cache object - * we have to store all the connections in a hash map in case there are multiple - * riak caches defined. +typedef struct mapcache_cache_riak mapcache_cache_riak; + +/**\class mapcache_cache_riak + * \brief a mapcache_cache for riak servers + * \implements mapcache_cache */ +struct mapcache_cache_riak { + mapcache_cache cache; + char *host; + int port; + char *key_template; + char *bucket_template; +}; struct riak_conn_params { mapcache_cache_riak *cache; }; -void mapcache_riak_connection_constructor(mapcache_context *ctx, void **conn_, void *params, apr_pool_t *pool) { +void mapcache_riak_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { mapcache_cache_riak *cache = ((struct riak_conn_params*)params)->cache; struct RIACK_CONNECTION_OPTIONS options; struct RIACK_CLIENT *client = riack_new_client(0); @@ -76,7 +84,7 @@ *conn_ = client; } -void mapcache_riak_connection_destructor(void *conn_, apr_pool_t *process_pool) { +void mapcache_riak_connection_destructor(void *conn_) { struct RIACK_CLIENT *client = (struct RIACK_CLIENT *)conn_; riack_free(client); } @@ -96,44 +104,34 @@ static int _mapcache_cache_riak_has_tile(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { int error; - int retries = 3; - RIACK_STRING key; + RIACK_STRING key,bucket; struct RIACK_GET_OBJECT obj; struct RIACK_CLIENT *client; mapcache_pooled_connection *pc; mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; - key.value = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + key.value = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); if (GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } key.len = strlen(key.value); + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + pc = _riak_get_connection(ctx, cache, tile); if (GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } client = pc->connection; - do - { - error = riack_get(client, cache->bucket, key, 0, &obj); - if (error != RIACK_SUCCESS) { - ctx->log(ctx, MAPCACHE_WARN, "Retry %d in riak_has_tile for tile %s from cache %s due to error %d", (4-retries), key.value, cache->cache.name, error); - for (error = riack_reconnect(client); - error != RIACK_SUCCESS && retries > 0; - error = riack_reconnect(client)) - { - --retries; - } - - --retries; - } - } - while (error != RIACK_SUCCESS && retries >= 0); + error = riack_get(client, bucket, key, 0, &obj); if (error != RIACK_SUCCESS) { - riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. mapcache_connection_pool_invalidate_connection(ctx,pc); ctx->set_error(ctx, 500, "riak: failed to get key %s: %d", key, error); return MAPCACHE_FALSE; @@ -153,7 +151,7 @@ static void _mapcache_cache_riak_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { int error; - RIACK_STRING key; + RIACK_STRING key,bucket; struct RIACK_CLIENT *client; struct RIACK_DEL_PROPERTIES properties; mapcache_pooled_connection *pc; @@ -162,17 +160,24 @@ memset(&properties, 0, sizeof(struct RIACK_DEL_PROPERTIES)); - key.value = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + key.value = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); GC_CHECK_ERROR(ctx); key.len = strlen(key.value); + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + pc = _riak_get_connection(ctx, cache, tile); GC_CHECK_ERROR(ctx); client = pc->connection; properties.rw_use = 1; properties.rw = (4294967295 - 3); // Special value meaning "ALL" - error = riack_delete(client, cache->bucket, key, &properties); + error = riack_delete(client, bucket, key, &properties); mapcache_connection_pool_release_connection(ctx,pc); @@ -190,9 +195,7 @@ */ static int _mapcache_cache_riak_get(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { int error; - int connect_error = RIACK_SUCCESS; - int retries = 3; - RIACK_STRING key; + RIACK_STRING key,bucket; struct RIACK_GET_OBJECT obj; struct RIACK_GET_PROPERTIES properties; struct RIACK_CLIENT *client; @@ -208,13 +211,19 @@ */ - key.value = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + key.value = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); if (GC_HAS_ERROR(ctx)) { return MAPCACHE_FAILURE; } key.len = strlen(key.value); - tile->encoded_data = mapcache_buffer_create(0, ctx->pool); + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + pc = _riak_get_connection(ctx, cache, tile); if (GC_HAS_ERROR(ctx)) { @@ -222,33 +231,11 @@ } client = pc->connection; - // If we get an error it is advised that we call reconnect. It also appears - // that every now and then we get an error and need to retry once again to - // get it to work. - do - { - error = riack_get(client, cache->bucket, key, &properties, &obj); - if (error != RIACK_SUCCESS) { - ctx->log(ctx, MAPCACHE_WARN, "Retry %d in riak_get for tile %s from cache %s due to error %d", (4-retries), key.value, cache->cache.name, error); - for (connect_error = riack_reconnect(client); - connect_error != RIACK_SUCCESS && retries > 0; - connect_error = riack_reconnect(client)) - { - --retries; - } - - --retries; - } - } - while (error != RIACK_SUCCESS && retries >= 0); + error = riack_get(client, bucket, key, &properties, &obj); if (error != RIACK_SUCCESS) { - if (connect_error != RIACK_SUCCESS) - mapcache_connection_pool_invalidate_connection(ctx,pc); - else - mapcache_connection_pool_release_connection(ctx,pc); - + mapcache_connection_pool_invalidate_connection(ctx,pc); ctx->set_error(ctx, 500, "Failed to get tile %s from cache %s due to error %d", key.value, cache->cache.name, error); return MAPCACHE_FAILURE; } @@ -263,6 +250,7 @@ } // Copy the data into the buffer + tile->encoded_data = mapcache_buffer_create(0, ctx->pool); mapcache_buffer_append(tile->encoded_data, obj.object.content[0].data_len, obj.object.content[0].data); riack_free_get_object(client, &obj); // riack_get allocates the returned object so we need to deallocate it. @@ -280,14 +268,13 @@ * \sa mapcache_cache::tile_set() */ static void _mapcache_cache_riak_set(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) { - char *key; + char *key,*content_type; int error; - int connect_error = RIACK_SUCCESS; - int retries = 3; struct RIACK_OBJECT object; struct RIACK_CONTENT content; struct RIACK_PUT_PROPERTIES properties; struct RIACK_CLIENT *client; + RIACK_STRING bucket; mapcache_pooled_connection *pc; mapcache_cache_riak *cache = (mapcache_cache_riak*)pcache; @@ -304,52 +291,50 @@ properties.dw = 0;*/ - key = mapcache_util_get_tile_key(ctx, tile, NULL, " \r\n\t\f\e\a\b", "#"); + key = mapcache_util_get_tile_key(ctx, tile, cache->key_template, " \r\n\t\f\e\a\b", "#"); GC_CHECK_ERROR(ctx); + if(strchr(cache->bucket_template,'{')) { + bucket.value = mapcache_util_get_tile_key(ctx, tile, cache->bucket_template, " \r\n\t\f\e\a\b", "#"); + } else { + bucket.value = cache->bucket_template; + } + bucket.len = strlen(bucket.value); + if (!tile->encoded_data) { tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); GC_CHECK_ERROR(ctx); } + content_type = tile->tileset->format?(tile->tileset->format->mime_type?tile->tileset->format->mime_type:NULL):NULL; + + if(!content_type) { + /* compute the content-type */ + mapcache_image_format_type t = mapcache_imageio_header_sniff(ctx,tile->encoded_data); + if(t == GC_PNG) + content_type = "image/png"; + else if(t == GC_JPEG) + content_type = "image/jpeg"; + } pc = _riak_get_connection(ctx, cache, tile); GC_CHECK_ERROR(ctx); client = pc->connection; // Set up the riak object to put. Need to do this after we get the client connection - object.bucket.value = cache->bucket.value; - object.bucket.len = cache->bucket.len; + object.bucket = bucket; object.key.value = key; object.key.len = strlen(key); object.vclock.len = 0; object.content_count = 1; object.content = &content; - content.content_type.value = tile->tileset->format->mime_type; - content.content_type.len = strlen(tile->tileset->format->mime_type); + content.content_type.value = content_type; + content.content_type.len = content_type?strlen(content_type):0; content.data = (uint8_t*)tile->encoded_data->buf; content.data_len = tile->encoded_data->size; - // If we get an error it is advised that we call reconnect. It also appears - // that every now and then we get an error and need to retry once again to - // get it to work. - do - { - error = riack_put(client, object, 0, &properties); - if (error != RIACK_SUCCESS) { - ctx->log(ctx, MAPCACHE_WARN, "Retry %d in riak_set for tile %s from cache %s due to eror %d", (4 - retries), key, cache->cache.name, error); - for (connect_error = riack_reconnect(client); - connect_error != RIACK_SUCCESS && retries > 0; - connect_error = riack_reconnect(client)) - { - --retries; - } + error = riack_put(client, object, 0, &properties); - --retries; - } - } - while (error != RIACK_SUCCESS && retries >= 0); - - if (connect_error != RIACK_SUCCESS) + if (error != RIACK_SUCCESS) mapcache_connection_pool_invalidate_connection(ctx,pc); else mapcache_connection_pool_release_connection(ctx,pc); @@ -364,7 +349,7 @@ * \private \memberof mapcache_cache_riak */ static void _mapcache_cache_riak_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_cache *cache, mapcache_cfg *config) { - ezxml_t cur_node,xhost,xport,xbucket; + ezxml_t cur_node,xhost,xport,xbucket,xkey; mapcache_cache_riak *dcache = (mapcache_cache_riak*)cache; int servercount = 0; @@ -386,16 +371,13 @@ xhost = ezxml_child(cur_node, "host"); /* Host should contain just server */ xport = ezxml_child(cur_node, "port"); xbucket = ezxml_child(cur_node, "bucket"); + xkey = ezxml_child(cur_node, "key"); if (!xhost || !xhost->txt || ! *xhost->txt) { ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); return; } else { dcache->host = apr_pstrdup(ctx->pool, xhost->txt); - if (dcache->host == NULL) { - ctx->set_error(ctx, 400, "cache %s: failed to allocate host string!", cache->name); - return; - } } if (!xport || !xport->txt || ! *xport->txt) { @@ -409,12 +391,11 @@ ctx->set_error(ctx, 400, "cache %s: with no ", cache->name); return; } else { - dcache->bucket.value = apr_pstrdup(ctx->pool, xbucket->txt); - if (dcache->bucket.value == NULL) { - ctx->set_error(ctx, 400, "cache %s: failed to allocate bucket string!", cache->name); - return; - } - dcache->bucket.len = strlen(dcache->bucket.value); + dcache->bucket_template = apr_pstrdup(ctx->pool, xbucket->txt); + } + + if(xkey && xkey->txt && *xkey->txt) { + dcache->key_template = apr_pstrdup(ctx->pool, xkey->txt); } } @@ -437,16 +418,23 @@ cache->cache.metadata = apr_table_make(ctx->pool, 3); cache->cache.type = MAPCACHE_CACHE_RIAK; - cache->cache.tile_get = _mapcache_cache_riak_get; - cache->cache.tile_exists = _mapcache_cache_riak_has_tile; - cache->cache.tile_set = _mapcache_cache_riak_set; - cache->cache.tile_delete = _mapcache_cache_riak_delete; + cache->cache._tile_get = _mapcache_cache_riak_get; + cache->cache._tile_exists = _mapcache_cache_riak_has_tile; + cache->cache._tile_set = _mapcache_cache_riak_set; + cache->cache._tile_delete = _mapcache_cache_riak_delete; cache->cache.configuration_parse_xml = _mapcache_cache_riak_configuration_parse_xml; cache->cache.configuration_post_config = _mapcache_cache_riak_configuration_post_config; cache->host = NULL; cache->port = 8087; // Default RIAK port used for protobuf + cache->bucket_template = NULL; + cache->key_template = NULL; return (mapcache_cache*)cache; } +#else +mapcache_cache* mapcache_cache_riak_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"RIAK support not compiled in this version"); + return NULL; +} #endif diff -Nru mapcache-1.4.0/lib/cache_sqlite.c mapcache-1.6.1/lib/cache_sqlite.c --- mapcache-1.4.0/lib/cache_sqlite.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_sqlite.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,10 +27,9 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_SQLITE -#include "mapcache.h" #include #include #include @@ -47,6 +46,36 @@ #include +/**\class mapcache_cache_sqlite + * \brief a mapcache_cache on a filesytem + * \implements mapcache_cache + */ +typedef struct mapcache_cache_sqlite mapcache_cache_sqlite; +typedef struct mapcache_cache_sqlite_stmt mapcache_cache_sqlite_stmt; + +struct mapcache_cache_sqlite_stmt { + char *sql; +}; + +struct sqlite_conn; + +struct mapcache_cache_sqlite { + mapcache_cache cache; + char *dbfile; + mapcache_cache_sqlite_stmt create_stmt; + mapcache_cache_sqlite_stmt exists_stmt; + mapcache_cache_sqlite_stmt get_stmt; + mapcache_cache_sqlite_stmt set_stmt; + mapcache_cache_sqlite_stmt delete_stmt; + apr_table_t *pragmas; + void (*bind_stmt)(mapcache_context *ctx, void *stmt, mapcache_cache_sqlite *cache, mapcache_tile *tile); + int n_prepared_statements; + int detect_blank; + char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; + int count_x, count_y; +}; + + struct sqlite_conn_params { mapcache_cache_sqlite *cache; char *dbfile; @@ -106,7 +135,7 @@ mapcache_connection_pool_release_connection(ctx,conn); } -void mapcache_sqlite_connection_constructor(mapcache_context *ctx, void **conn_, void *params, apr_pool_t *process_pool) +void mapcache_sqlite_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; int flags; @@ -117,6 +146,8 @@ flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; } else { flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_CREATE; + mapcache_make_parent_dirs(ctx,sq_params->dbfile); + GC_CHECK_ERROR(ctx); } ret = sqlite3_open_v2(sq_params->dbfile, &conn->handle, flags, NULL); if (ret != SQLITE_OK) { @@ -144,7 +175,7 @@ conn->nstatements = sq_params->cache->n_prepared_statements; } -void mapcache_sqlite_connection_destructor(void *conn_, apr_pool_t *process_pool) +void mapcache_sqlite_connection_destructor(void *conn_) { struct sqlite_conn *conn = (struct sqlite_conn*) conn_; int i; @@ -196,14 +227,17 @@ if(strstr(*path,"{grid}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); - if(tile->dimensions && strstr(*path,"{dim}")) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->val,"/.",'#'); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->cached_value,"/.",'#'); + char *single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); dimstring = apr_pstrcat(ctx->pool,dimstring,"#",dimval,NULL); + if(strstr(*path,single_dim)) { + *path = mapcache_util_str_replace(ctx->pool,*path, single_dim, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -953,11 +987,11 @@ } cache->cache.metadata = apr_table_make(ctx->pool, 3); cache->cache.type = MAPCACHE_CACHE_SQLITE; - cache->cache.tile_delete = _mapcache_cache_sqlite_delete; - cache->cache.tile_get = _mapcache_cache_sqlite_get; - cache->cache.tile_exists = _mapcache_cache_sqlite_has_tile; - cache->cache.tile_set = _mapcache_cache_sqlite_set; - cache->cache.tile_multi_set = _mapcache_cache_sqlite_multi_set; + cache->cache._tile_delete = _mapcache_cache_sqlite_delete; + cache->cache._tile_get = _mapcache_cache_sqlite_get; + cache->cache._tile_exists = _mapcache_cache_sqlite_has_tile; + cache->cache._tile_set = _mapcache_cache_sqlite_set; + cache->cache._tile_multi_set = _mapcache_cache_sqlite_multi_set; cache->cache.configuration_post_config = _mapcache_cache_sqlite_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_sqlite_configuration_parse_xml; cache->create_stmt.sql = apr_pstrdup(ctx->pool, @@ -992,9 +1026,9 @@ return NULL; } cache->cache.configuration_post_config = _mapcache_cache_mbtiles_configuration_post_config; - cache->cache.tile_set = _mapcache_cache_mbtiles_set; - cache->cache.tile_multi_set = _mapcache_cache_mbtiles_multi_set; - cache->cache.tile_delete = _mapcache_cache_mbtiles_delete; + cache->cache._tile_set = _mapcache_cache_mbtiles_set; + cache->cache._tile_multi_set = _mapcache_cache_mbtiles_multi_set; + cache->cache._tile_delete = _mapcache_cache_mbtiles_delete; cache->create_stmt.sql = apr_pstrdup(ctx->pool, "create table if not exists images(tile_id text, tile_data blob, primary key(tile_id));"\ "CREATE TABLE IF NOT EXISTS map (zoom_level integer, tile_column integer, tile_row integer, tile_id text, foreign key(tile_id) references images(tile_id), primary key(tile_row,tile_column,zoom_level));"\ @@ -1011,6 +1045,17 @@ cache->bind_stmt = _bind_mbtiles_params; return (mapcache_cache*) cache; } +#else + +mapcache_cache* mapcache_cache_sqlite_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"SQLITE support not compiled in this version"); + return NULL; +} +mapcache_cache* mapcache_cache_mbtiles_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"SQLITE (MBtiles) support not compiled in this version"); + return NULL; +} + #endif diff -Nru mapcache-1.4.0/lib/cache_tiff.c mapcache-1.6.1/lib/cache_tiff.c --- mapcache-1.4.0/lib/cache_tiff.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_tiff.c 2017-09-29 21:31:13.000000000 +0000 @@ -3,10 +3,13 @@ * * Project: MapServer * Purpose: MapCache tile caching: tiled tiff filesytem cache backend. - * Author: Thomas Bonfort, Frank Warmerdam and the MapServer team. + * Author: Thomas Bonfort + * Frank Warmerdam + * Even Rouault + * MapServer team. * ****************************************************************************** - * Copyright (c) 2011 Regents of the University of Minnesota. + * Copyright (c) 2011-2017 Regents of the University of Minnesota. * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -27,10 +30,9 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ -#include "mapcache-config.h" +#include "mapcache.h" #ifdef USE_TIFF -#include "mapcache.h" #include #include #include @@ -39,6 +41,12 @@ #include #include +#ifdef USE_GDAL +#include "cpl_vsi.h" +#include "cpl_conv.h" +#define CPL_SERV_H_INCLUDED +#endif + #ifdef USE_GEOTIFF #include "xtiffio.h" #include "geovalues.h" @@ -53,6 +61,278 @@ #define MyTIFFClose TIFFClose #endif +typedef enum +{ + MAPCACHE_TIFF_STORAGE_FILE, + MAPCACHE_TIFF_STORAGE_REST, + MAPCACHE_TIFF_STORAGE_GOOGLE +} mapcache_cache_tiff_storage_type; + + +typedef struct mapcache_cache_tiff mapcache_cache_tiff; + +struct mapcache_cache_tiff { + mapcache_cache cache; + char *filename_template; + char *x_fmt,*y_fmt,*z_fmt,*inv_x_fmt,*inv_y_fmt,*div_x_fmt,*div_y_fmt,*inv_div_x_fmt,*inv_div_y_fmt; + int count_x; + int count_y; + mapcache_image_format_jpeg *format; + mapcache_locker *locker; + struct { + mapcache_cache_tiff_storage_type type; + int connection_timeout; + int timeout; + char *header_file; + union + { + struct + { + char* access; + char* secret; + } google; + } u; + } storage; +}; + +#ifdef USE_GDAL + +static tsize_t +_tiffReadProc( thandle_t th, tdata_t buf, tsize_t size ) +{ + VSILFILE* fp = (VSILFILE*)th; + return VSIFReadL( buf, 1, size, fp ); +} + +static tsize_t +_tiffWriteProc( thandle_t th, tdata_t buf, tsize_t size ) +{ + VSILFILE* fp = (VSILFILE*)th; + return VSIFWriteL( buf, 1, size, fp ); +} + +static toff_t +_tiffSeekProc( thandle_t th, toff_t off, int whence ) +{ + VSILFILE* fp = (VSILFILE*)th; + if( VSIFSeekL( fp, off, whence ) != 0 ) + { + TIFFErrorExt( th, "_tiffSeekProc", "%s", VSIStrerror( errno ) ); + return (toff_t)( -1 ); + } + return VSIFTellL( fp ); +} + +static int +_tiffCloseProc( thandle_t th ) +{ + VSILFILE* fp = (VSILFILE*)th; + VSIFCloseL(fp); + return 0; +} + +static toff_t +_tiffSizeProc( thandle_t th ) +{ + vsi_l_offset old_off; + toff_t file_size; + VSILFILE* fp = (VSILFILE*)th; + + old_off = VSIFTellL( fp ); + (void)VSIFSeekL( fp, 0, SEEK_END ); + + file_size = (toff_t) VSIFTellL( fp ); + (void)VSIFSeekL( fp, old_off, SEEK_SET ); + + return file_size; +} + +static int +_tiffMapProc( thandle_t th, tdata_t* pbase, toff_t* psize ) +{ + (void)th; + (void)pbase; + (void)psize; + /* Unimplemented */ + return 0; +} + +static void +_tiffUnmapProc( thandle_t th, tdata_t base, toff_t size ) +{ + (void)th; + (void)base; + (void)size; + /* Unimplemented */ +} + +static char* set_conf_value(const char* key, const char* value) +{ + const char* old_val_const; + char* old_val = NULL; + old_val_const = CPLGetConfigOption(key, NULL); + if( old_val_const != NULL ) + old_val = strdup(old_val_const); + /* Prevent a directory listing to be done */ + CPLSetConfigOption(key, value); + return old_val; +} + +static void restore_conf_value(const char* key, char* old_val) +{ + CPLSetConfigOption(key, old_val); + free(old_val); +} + +typedef struct +{ + char* old_val_disable_readdir; + char* old_val_headerfile; + char* old_val_secret; + char* old_val_access; +} mapache_gdal_env_context; + +static void set_gdal_context(mapcache_cache_tiff *cache, + mapache_gdal_env_context* pcontext) +{ + memset(pcontext, 0, sizeof(mapache_gdal_env_context)); + /* Prevent a directory listing to be done */ + pcontext->old_val_disable_readdir = + set_conf_value("GDAL_DISABLE_READDIR_ON_OPEN", "YES"); + + if( cache->storage.header_file ) { + pcontext->old_val_headerfile = set_conf_value("GDAL_HTTP_HEADER_FILE", + cache->storage.header_file); + } + if( cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE ) { + pcontext->old_val_secret = set_conf_value("GS_SECRET_ACCESS_KEY", + cache->storage.u.google.secret); + pcontext->old_val_access = set_conf_value("GS_ACCESS_KEY_ID", + cache->storage.u.google.access); + } +} + +static void restore_gdal_context(mapcache_cache_tiff *cache, + mapache_gdal_env_context* pcontext) +{ + + restore_conf_value("GDAL_DISABLE_READDIR_ON_OPEN", + pcontext->old_val_disable_readdir); + if( cache->storage.header_file ) { + restore_conf_value("GDAL_HTTP_HEADER_FILE", + pcontext->old_val_headerfile); + } + if( cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE ) { + restore_conf_value("GS_SECRET_ACCESS_KEY", + pcontext->old_val_secret); + restore_conf_value("GS_ACCESS_KEY_ID", + pcontext->old_val_access); + } +} + +static int mapcache_cache_tiff_vsi_stat( + mapcache_cache_tiff *cache, + const char* name, + VSIStatBufL* pstat) +{ + mapache_gdal_env_context context; + int ret; + + set_gdal_context(cache, &context); + ret = VSIStatL(name, pstat); + restore_gdal_context(cache, &context); + + return ret; +} + +static VSILFILE* mapcache_cache_tiff_vsi_open( + mapcache_cache_tiff *cache, + const char* name, const char* mode ) +{ + mapache_gdal_env_context context; + VSILFILE* fp; + + set_gdal_context(cache, &context); + fp = VSIFOpenL(name, mode); + restore_gdal_context(cache, &context); + + return fp; +} + +static TIFF* mapcache_cache_tiff_open(mapcache_context *ctx, + mapcache_cache_tiff *cache, + const char* name, const char* mode ) +{ + char chDummy; + VSILFILE* fp; + + /* If writing or using a regular filename, then use standard */ + /* libtiff/libgeotiff I/O layer */ + if( strcmp(mode, "r") != 0 || strncmp(name, "/vsi", 4) != 0 ) + { + return MyTIFFOpen(name, mode); + } + + fp = mapcache_cache_tiff_vsi_open(cache, name, mode); + if( fp == NULL ) + return NULL; + + if( strcmp(mode, "r") == 0 ) + { + /* But then the file descriptor may point to an invalid resource */ + /* so try reading a byte from it */ + if(VSIFReadL(&chDummy, 1, 1, fp) != 1) + { + VSIFCloseL(fp); + return NULL; + } + VSIFSeekL(fp, 0, SEEK_SET); + } + + return +#ifdef USE_GEOTIFF + XTIFFClientOpen +#else + TIFFClientOpen +#endif + ( name, mode, + (thandle_t) fp, + _tiffReadProc, _tiffWriteProc, + _tiffSeekProc, _tiffCloseProc, _tiffSizeProc, + _tiffMapProc, _tiffUnmapProc ); +} + +static void CPL_STDCALL mapcache_cache_tiff_gdal_error_handler(CPLErr eErr, + int error_num, + const char* pszMsg) +{ +#ifdef DEBUG + mapcache_context *ctx = (mapcache_context *) CPLGetErrorHandlerUserData(); + ctx->log(ctx,MAPCACHE_DEBUG,"GDAL %s, %d: %s", + (eErr == CE_Failure) ? "Failure": + (eErr == CE_Warning) ? "Warning": + "Debug", + error_num, + pszMsg); +#endif +} + + +#else + +static TIFF* mapcache_cache_tiff_open(mapcache_context *ctx, + mapcache_cache_tiff *cache, + const char* name, const char* mode ) +{ + (void)ctx; + (void)cache; + return MyTIFFOpen(name, mode); + +} + +#endif /* USE_GDAL */ + +#undef MyTIFFOpen /** * \brief return filename for given tile @@ -64,7 +344,20 @@ */ static void _mapcache_cache_tiff_tile_key(mapcache_context *ctx, mapcache_cache_tiff *cache, mapcache_tile *tile, char **path) { - *path = cache->filename_template; + if( cache->storage.type == MAPCACHE_TIFF_STORAGE_REST ) { + *path = apr_pstrcat(ctx->pool, "/vsicurl/", cache->filename_template, NULL); + } + else if( cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE && + strncmp(cache->filename_template, + "https://storage.googleapis.com/", + strlen("https://storage.googleapis.com/")) == 0) { + *path = apr_pstrcat(ctx->pool, "/vsigs/", + cache->filename_template + + strlen("https://storage.googleapis.com/"), NULL); + } + else { + *path = apr_pstrdup(ctx->pool, cache->filename_template); + } /* * generic template substitutions @@ -75,14 +368,17 @@ if(strstr(*path,"{grid}")) *path = mapcache_util_str_replace(ctx->pool,*path, "{grid}", tile->grid_link->grid->name); - if(tile->dimensions && strstr(*path,"{dim}")) { + if(tile->dimensions && strstr(*path,"{dim")) { char *dimstring=""; - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); - const char *dimval = mapcache_util_str_sanitize(ctx->pool,entry->val,"/.",'#'); + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + const char *dimval = mapcache_util_str_sanitize(ctx->pool,rdim->cached_value,"/.",'#'); + char *dim_key = apr_pstrcat(ctx->pool,"{dim:",rdim->dimension->name,"}",NULL); dimstring = apr_pstrcat(ctx->pool,dimstring,"#",dimval,NULL); + if(strstr(*path,dim_key)) { + *path = mapcache_util_str_replace(ctx->pool,*path, dim_key, dimval); + } } *path = mapcache_util_str_replace(ctx->pool,*path, "{dim}", dimstring); } @@ -220,7 +516,12 @@ if(GC_HAS_ERROR(ctx)) { return MAPCACHE_FALSE; } - hTIFF = MyTIFFOpen(filename,"r"); + +#ifdef USE_GDAL + CPLPushErrorHandlerEx(mapcache_cache_tiff_gdal_error_handler, ctx); +#endif + + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"r"); if(hTIFF) { do { @@ -246,6 +547,9 @@ check_tiff_format(ctx,cache,tile,hTIFF,filename); if(GC_HAS_ERROR(ctx)) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } #endif @@ -265,22 +569,37 @@ if(1 != TIFFGetField( hTIFF, TIFFTAG_TILEOFFSETS, &offsets )) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } if(1 != TIFFGetField( hTIFF, TIFFTAG_TILEBYTECOUNTS, &sizes )) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } MyTIFFClose(hTIFF); if( offsets[tiff_off] > 0 && sizes[tiff_off] > 0 ) { +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_TRUE; } else { +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FALSE; } } while( TIFFReadDirectory( hTIFF ) ); - return MAPCACHE_FALSE; /* TIFF only contains overviews ? */ - } else - return MAPCACHE_FALSE; + /* TIFF only contains overviews ? */ + } +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif + return MAPCACHE_FALSE; } static void _mapcache_cache_tiff_delete(mapcache_context *ctx, mapcache_cache *pcache, mapcache_tile *tile) @@ -311,13 +630,17 @@ tile->x,tile->y,tile->z,filename); #endif - hTIFF = MyTIFFOpen(filename,"r"); +#ifdef USE_GDAL + CPLPushErrorHandlerEx(mapcache_cache_tiff_gdal_error_handler, ctx); +#endif + + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"r"); /* * we currrently have no way of knowing if the opening failed because the tif * file does not exist (which is not an error condition, as it only signals * that the requested tile does not exist in the cache), or if an other error - * that should be signaled occured (access denied, not a tiff file, etc...) + * that should be signaled occurred (access denied, not a tiff file, etc...) * * we ignore this case here and hope that further parts of the code will be * able to detect what's happening more precisely @@ -346,6 +669,9 @@ check_tiff_format(ctx,cache,tile,hTIFF,filename); if(GC_HAS_ERROR(ctx)) { MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } #endif @@ -374,6 +700,9 @@ ctx->set_error(ctx,500,"Failed to read TIFF file \"%s\" tile offsets", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -383,6 +712,9 @@ ctx->set_error(ctx,500,"Failed to read TIFF file \"%s\" tile sizes", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -391,7 +723,7 @@ * are not zero for that index. * if not, the tiff file is sparse and is missing the requested tile */ - if( offsets[tiff_off] > 0 && sizes[tiff_off] > 0 ) { + if( offsets[tiff_off] > 0 && sizes[tiff_off] >= 2 ) { apr_file_t *f; apr_finfo_t finfo; apr_status_t ret; @@ -400,11 +732,14 @@ uint32 jpegtable_size = 0; unsigned char* jpegtable_ptr; rv = TIFFGetField( hTIFF, TIFFTAG_JPEGTABLES, &jpegtable_size, &jpegtable_ptr ); - if( rv != 1 || !jpegtable_ptr || !jpegtable_size) { + if( rv != 1 || !jpegtable_ptr || jpegtable_size < 2) { /* there is no common jpeg header in the tiff tags */ ctx->set_error(ctx,500,"Failed to read TIFF file \"%s\" jpeg table", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -412,6 +747,90 @@ * open the tiff file directly to access the jpeg image data with the given * offset */ +#ifdef USE_GDAL + if( cache->storage.type != MAPCACHE_TIFF_STORAGE_FILE ) + { + char *bufptr; + apr_off_t off; + apr_size_t bytes_to_read; + size_t bytes_read; + VSIStatBufL sStat; + VSILFILE* fp; + + fp = mapcache_cache_tiff_vsi_open(cache, filename, "r"); + if( fp == NULL ) + { + /* + * shouldn't usually happen. we managed to open the file before, + * nothing much to do except bail out. + */ + ctx->set_error(ctx,500, + "VSIFOpenL() failed on already open tiff " + "file \"%s\", giving up .... ", + filename); + MyTIFFClose(hTIFF); + CPLPopErrorHandler(); + return MAPCACHE_FAILURE; + } + + if( mapcache_cache_tiff_vsi_stat(cache, filename, &sStat) == 0 ) { + /* + * extract the file modification time. this isn't guaranteed to be the + * modification time of the actual tile, but it's the best we can do + */ + tile->mtime = sStat.st_mtime; + } + +#ifdef DEBUG + ctx->log(ctx,MAPCACHE_DEBUG,"tile (%d,%d,%d) => mtime = %d)", + tile->x,tile->y,tile->z,tile->mtime); +#endif + + + /* create a memory buffer to contain the jpeg data */ + tile->encoded_data = mapcache_buffer_create( + (jpegtable_size+sizes[tiff_off]-4),ctx->pool); + + /* + * copy the jpeg header to the beginning of the memory buffer, + * omitting the last 2 bytes + */ + memcpy(tile->encoded_data->buf,jpegtable_ptr,(jpegtable_size-2)); + + /* advance the data pointer to after the header data */ + bufptr = ((char *)tile->encoded_data->buf) + (jpegtable_size-2); + + + /* go to the specified offset in the tiff file, plus 2 bytes */ + off = offsets[tiff_off]+2; + VSIFSeekL(fp, (vsi_l_offset)off, SEEK_SET); + + /* + * copy the jpeg body at the end of the memory buffer, accounting + * for the two bytes we omitted in the previous step + */ + bytes_to_read = sizes[tiff_off]-2; + bytes_read = VSIFReadL(bufptr, 1, bytes_to_read, fp); + + /* check we have correctly read the requested number of bytes */ + if(bytes_to_read != bytes_read) { + ctx->set_error(ctx,500,"failed to read jpeg body in \"%s\".\ + (read %d of %d bytes)", + filename,(int)bytes_read,(int)sizes[tiff_off]-2); + VSIFCloseL(fp); + MyTIFFClose(hTIFF); + CPLPopErrorHandler(); + return MAPCACHE_FAILURE; + } + + tile->encoded_data->size = (jpegtable_size+sizes[tiff_off]-4); + + VSIFCloseL(fp); + CPLPopErrorHandler(); + return MAPCACHE_SUCCESS; + } + else +#endif if((ret=apr_file_open(&f, filename, APR_FOPEN_READ|APR_FOPEN_BUFFERED|APR_FOPEN_BINARY,APR_OS_DEFAULT, ctx->pool)) == APR_SUCCESS) { @@ -437,7 +856,7 @@ memcpy(tile->encoded_data->buf,jpegtable_ptr,(jpegtable_size-2)); /* advance the data pointer to after the header data */ - bufptr = tile->encoded_data->buf + (jpegtable_size-2); + bufptr = ((char *)tile->encoded_data->buf) + (jpegtable_size-2); /* go to the specified offset in the tiff file, plus 2 bytes */ @@ -455,7 +874,11 @@ if(bytes != sizes[tiff_off]-2) { ctx->set_error(ctx,500,"failed to read jpeg body in \"%s\".\ (read %d of %d bytes)", filename,bytes,sizes[tiff_off]-2); + apr_file_close(f); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } @@ -464,6 +887,9 @@ /* finalize and cleanup */ apr_file_close(f); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_SUCCESS; } else { /* shouldn't usually happen. we managed to open the file with TIFFOpen, @@ -473,11 +899,17 @@ ctx->set_error(ctx,500,"apr_file_open failed on already open tiff file \"%s\", giving up .... ", filename); MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_FAILURE; } } else { /* sparse tiff file without the requested tile */ MyTIFFClose(hTIFF); +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif return MAPCACHE_CACHE_MISS; } } /* loop through the tiff directories if there are multiple ones */ @@ -489,8 +921,10 @@ * does the file only contain overviews? */ MyTIFFClose(hTIFF); - return MAPCACHE_CACHE_MISS; } +#ifdef USE_GDAL + CPLPopErrorHandler(); +#endif /* failed to open tiff file */ return MAPCACHE_CACHE_MISS; } @@ -512,10 +946,8 @@ int rv; void *lock; int create; - char errmsg[120]; mapcache_cache_tiff *cache; mapcache_image_format_jpeg *format; - char *hackptr1,*hackptr2; int tilew; int tileh; unsigned char *rgb; @@ -528,6 +960,11 @@ int tiff_off; /* the index of the tile inside the list of tiles of the tiff image */ cache = (mapcache_cache_tiff*)pcache; + if( cache->storage.type != MAPCACHE_TIFF_STORAGE_FILE ) + { + ctx->set_error(ctx,500,"tiff cache %s is read-only\n",pcache->name); + return; + } _mapcache_cache_tiff_tile_key(ctx, cache, tile, &filename); format = (mapcache_image_format_jpeg*) cache->format; if(GC_HAS_ERROR(ctx)) { @@ -541,27 +978,8 @@ /* * create the directory where the tiff file will be stored */ - - /* find the location of the last '/' in the string */ - hackptr2 = hackptr1 = filename; - while(*hackptr1) { - if(*hackptr1 == '/') - hackptr2 = hackptr1; - hackptr1++; - } - *hackptr2 = '\0'; - - if(APR_SUCCESS != (rv = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool))) { - /* - * apr_dir_make_recursive sometimes sends back this error, although it should not. - * ignore this one - */ - if(!APR_STATUS_IS_EEXIST(rv)) { - ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(rv,errmsg,120)); - return; - } - } - *hackptr2 = '/'; + mapcache_make_parent_dirs(ctx,filename); + GC_CHECK_ERROR(ctx); tilew = tile->grid_link->grid->tile_sx; tileh = tile->grid_link->grid->tile_sy; @@ -594,10 +1012,10 @@ /* check if the tiff file exists already */ rv = apr_stat(&finfo,filename,0,ctx->pool); if(!APR_STATUS_IS_ENOENT(rv)) { - hTIFF = MyTIFFOpen(filename,"r+"); + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"r+"); create = 0; } else { - hTIFF = MyTIFFOpen(filename,"w+"); + hTIFF = mapcache_cache_tiff_open(ctx,cache,filename,"w+"); create = 1; } if(!hTIFF) { @@ -680,7 +1098,7 @@ x = (tile->x / cache->count_x)*(cache->count_x); y = (tile->y / cache->count_y)*(cache->count_y) + ntilesy - 1; - mapcache_grid_get_extent(ctx, tile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, tile->grid_link->grid, x,y,tile->z,&bbox); adfTiePoints[0] = 0.0; adfTiePoints[1] = 0.0; @@ -735,7 +1153,7 @@ close_tiff: if(hTIFF) MyTIFFClose(hTIFF); - mapcache_unlock_resource(ctx,cache->locker?cache->locker:ctx->config->locker,filename, lock); + mapcache_unlock_resource(ctx,cache->locker?cache->locker:ctx->config->locker, lock); #else ctx->set_error(ctx,500,"tiff write support disabled by default"); #endif @@ -833,7 +1251,87 @@ if(cur_node) { mapcache_config_parse_locker(ctx, cur_node, &cache->locker); } - + + cache->storage.type = MAPCACHE_TIFF_STORAGE_FILE; + cur_node = ezxml_child(node,"storage"); + if (cur_node) { + ezxml_t child_node; + const char *type = ezxml_attr(cur_node,"type"); + if( !type ) { + ctx->set_error(ctx,400, + " with no \"type\" attribute in cache (%s)", + pcache->name); + return; + } + + if( strcmp(type, "rest") == 0 ) { + cache->storage.type = MAPCACHE_TIFF_STORAGE_REST; + } + else if( strcmp(type, "google") == 0 ) { + cache->storage.type = MAPCACHE_TIFF_STORAGE_GOOGLE; + if ((child_node = ezxml_child(cur_node,"access")) != NULL) { + cache->storage.u.google.access = + apr_pstrdup(ctx->pool, child_node->txt); + } else if ( getenv("GS_ACCESS_KEY_ID")) { + cache->storage.u.google.access = + apr_pstrdup(ctx->pool,getenv("GS_ACCESS_KEY_ID")); + } else { + ctx->set_error(ctx,400, + "google storage in cache (%s) is missing " + "required child", pcache->name); + return; + } + if ((child_node = ezxml_child(cur_node,"secret")) != NULL) { + cache->storage.u.google.secret = + apr_pstrdup(ctx->pool, child_node->txt); + } else if ( getenv("GS_SECRET_ACCESS_KEY")) { + cache->storage.u.google.access = + apr_pstrdup(ctx->pool,getenv("GS_SECRET_ACCESS_KEY")); + } else { + ctx->set_error(ctx,400, + "google storage in cache (%s) is missing " + "required child", pcache->name); + return; + } + } + else { + ctx->set_error(ctx, 400, "unknown cache type %s for cache \"%s\"", + type, pcache->name); + return; + } + + if ((child_node = ezxml_child(cur_node,"connection_timeout")) != NULL) { + char *endptr; + cache->storage.connection_timeout = (int)strtol(child_node->txt,&endptr,10); + if(*endptr != 0 || cache->storage.connection_timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache " + "\"%s\" (positive integer expected)", + child_node->txt); + return; + } + } else { + cache->storage.connection_timeout = 30; + } + + if ((child_node = ezxml_child(cur_node,"timeout")) != NULL) { + char *endptr; + cache->storage.timeout = (int)strtol(child_node->txt,&endptr,10); + if(*endptr != 0 || cache->storage.timeout<1) { + ctx->set_error(ctx,400,"invalid rest cache \"%s\" " + "(positive integer expected)", + child_node->txt); + return; + } + } else { + cache->storage.timeout = 120; + } + + if ((child_node = ezxml_child(cur_node,"header_file")) != NULL) { + cache->storage.header_file = apr_pstrdup(ctx->pool, child_node->txt); + } + + } + } /** @@ -852,6 +1350,33 @@ ctx->set_error(ctx, 400, "tiff cache %s has invalid count (%d,%d)",cache->count_x,cache->count_y); return; } + +#ifdef USE_GDAL + if(cache->storage.type == MAPCACHE_TIFF_STORAGE_REST && + strncmp(cache->filename_template, "http://", 6) != 0 && + strncmp(cache->filename_template, "https://", 7) != 0 ) { + ctx->set_error(ctx, 400, "tiff cache %s template pattern should begin with http:// or https://",cache->cache.name); + return; + } + + if(cache->storage.type == MAPCACHE_TIFF_STORAGE_GOOGLE && + strncmp(cache->filename_template, "https://storage.googleapis.com/", + strlen("https://storage.googleapis.com/")) != 0 && + strncmp(cache->filename_template, "/vsigs/", + strlen("/vsigs/")) != 0 ) { + ctx->set_error(ctx, 400, "tiff cache %s template pattern should begin " + "with https://storage.googleapis.com/ or /vsigs/",cache->cache.name); + return; + } +#else + if(cache->storage.type != MAPCACHE_TIFF_STORAGE_FILE ) + { + ctx->set_error(ctx, 400, "tiff cache %s cannot use a network based " + "storage due to mising GDAL dependency", + cache->cache.name); + return; + } +#endif } /** @@ -866,10 +1391,10 @@ } cache->cache.metadata = apr_table_make(ctx->pool,3); cache->cache.type = MAPCACHE_CACHE_TIFF; - cache->cache.tile_delete = _mapcache_cache_tiff_delete; - cache->cache.tile_get = _mapcache_cache_tiff_get; - cache->cache.tile_exists = _mapcache_cache_tiff_has_tile; - cache->cache.tile_set = _mapcache_cache_tiff_set; + cache->cache._tile_delete = _mapcache_cache_tiff_delete; + cache->cache._tile_get = _mapcache_cache_tiff_get; + cache->cache._tile_exists = _mapcache_cache_tiff_has_tile; + cache->cache._tile_set = _mapcache_cache_tiff_set; cache->cache.configuration_post_config = _mapcache_cache_tiff_configuration_post_config; cache->cache.configuration_parse_xml = _mapcache_cache_tiff_configuration_parse_xml; cache->count_x = 10; @@ -885,6 +1410,13 @@ return (mapcache_cache*)cache; } +#else + +mapcache_cache* mapcache_cache_tiff_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"TIFF support not compiled in this version"); + return NULL; +} + #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.4.0/lib/cache_tokyocabinet.c mapcache-1.6.1/lib/cache_tokyocabinet.c --- mapcache-1.4.0/lib/cache_tokyocabinet.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/cache_tokyocabinet.c 2017-09-29 21:31:13.000000000 +0000 @@ -27,9 +27,9 @@ * DEALINGS IN THE SOFTWARE. *****************************************************************************/ +#include "mapcache.h" #ifdef USE_TC -#include "mapcache.h" #include #include #include @@ -43,6 +43,14 @@ #include #endif +typedef struct mapcache_cache_tc mapcache_cache_tc; +struct mapcache_cache_tc { + mapcache_cache cache; + char *basedir; + char *key_template; + mapcache_context *ctx; +}; +mapcache_cache *mapcache_cache_tc_create(mapcache_context *ctx); struct tc_conn { TCBDB *bdb; @@ -216,6 +224,13 @@ return (mapcache_cache*)cache; } +#else + +mapcache_cache* mapcache_cache_tc_create(mapcache_context *ctx) { + ctx->set_error(ctx,400,"TokyoCabinet support not compiled in this version"); + return NULL; +} + #endif /* vim: ts=2 sts=2 et sw=2 diff -Nru mapcache-1.4.0/lib/configuration.c mapcache-1.6.1/lib/configuration.c --- mapcache-1.4.0/lib/configuration.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/configuration.c 2017-09-29 21:31:13.000000000 +0000 @@ -135,12 +135,12 @@ mapcache_imageio_create_png_q_format(pool,"PNG8",MAPCACHE_COMPRESSION_FAST,256), "PNG8"); mapcache_configuration_add_image_format(cfg, - mapcache_imageio_create_jpeg_format(pool,"JPEG",90,MAPCACHE_PHOTOMETRIC_YCBCR), + mapcache_imageio_create_jpeg_format(pool,"JPEG",90,MAPCACHE_PHOTOMETRIC_YCBCR,MAPCACHE_OPTIMIZE_YES), "JPEG"); mapcache_configuration_add_image_format(cfg, mapcache_imageio_create_mixed_format(pool,"mixed", mapcache_configuration_get_image_format(cfg,"PNG"), - mapcache_configuration_get_image_format(cfg,"JPEG")), + mapcache_configuration_get_image_format(cfg,"JPEG"), 255), "mixed"); cfg->default_image_format = mapcache_configuration_get_image_format(cfg,"mixed"); cfg->reporting = MAPCACHE_REPORT_MSG; diff -Nru mapcache-1.4.0/lib/configuration_xml.c mapcache-1.6.1/lib/configuration_xml.c --- mapcache-1.4.0/lib/configuration_xml.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/configuration_xml.c 2017-09-29 21:31:13.000000000 +0000 @@ -47,47 +47,6 @@ } } -void parseTimeDimension(mapcache_context *ctx, ezxml_t node, mapcache_tileset *tileset) { - const char *attr = NULL; - if(tileset->timedimension) { - ctx->set_error(ctx,400,"tileset \"%s\" can only have a single ", tileset->name); - return; - } - attr = ezxml_attr(node,"type"); - if(attr && *attr) { - if(!strcmp(attr,"sqlite")) { -#ifdef USE_SQLITE - tileset->timedimension = mapcache_timedimension_sqlite_create(ctx->pool); -#else - ctx->set_error(ctx,400, "failed to add sqlite timedimension: Sqlite support is not available on this build"); - return; -#endif - } else { - ctx->set_error(ctx,400,"unknown \"type\" attribute \"%s\" for %s's . Expecting one of (sqlite)",attr,tileset->name); - return; - } - - } else { - ctx->set_error(ctx,400,"missing \"type\" attribute for %s's ",tileset->name); - return; - } - attr = ezxml_attr(node,"name"); - if(attr && *attr) { - tileset->timedimension->key = apr_pstrdup(ctx->pool,attr); - } else { - tileset->timedimension->key = apr_pstrdup(ctx->pool,"TIME"); - } - - attr = ezxml_attr(node,"default"); - if(attr && *attr) { - tileset->timedimension->default_value = apr_pstrdup(ctx->pool,attr); - } else { - ctx->set_error(ctx,400,"no \"default\" attribute for %s",tileset->timedimension->key); - return; - } - tileset->timedimension->configuration_parse_xml(ctx,tileset->timedimension,node); -} - void parseDimensions(mapcache_context *ctx, ezxml_t node, mapcache_tileset *tileset) { ezxml_t dimension_node; @@ -96,7 +55,6 @@ char *name = (char*)ezxml_attr(dimension_node,"name"); char *type = (char*)ezxml_attr(dimension_node,"type"); char *unit = (char*)ezxml_attr(dimension_node,"unit"); - char *skip_validation = (char*)ezxml_attr(dimension_node,"skip_validation"); char *default_value = (char*)ezxml_attr(dimension_node,"default"); mapcache_dimension *dimension = NULL; @@ -108,17 +66,13 @@ if(type && *type) { if(!strcmp(type,"values")) { - dimension = mapcache_dimension_values_create(ctx->pool); + dimension = mapcache_dimension_values_create(ctx,ctx->pool); } else if(!strcmp(type,"regex")) { - dimension = mapcache_dimension_regex_create(ctx->pool); - } else if(!strcmp(type,"intervals")) { - dimension = mapcache_dimension_intervals_create(ctx->pool); + dimension = mapcache_dimension_regex_create(ctx,ctx->pool); } else if(!strcmp(type,"sqlite")) { - dimension = mapcache_dimension_sqlite_create(ctx->pool); + dimension = mapcache_dimension_sqlite_create(ctx,ctx->pool); } else if(!strcmp(type,"time")) { - ctx->set_error(ctx,501,"time dimension type not implemented yet"); - return; - dimension = mapcache_dimension_time_create(ctx->pool); + dimension = mapcache_dimension_time_create(ctx,ctx->pool); } else { ctx->set_error(ctx,400,"unknown dimension type \"%s\"",type); return; @@ -127,6 +81,7 @@ ctx->set_error(ctx,400, "mandatory attribute \"type\" not found in "); return; } + GC_CHECK_ERROR(ctx); dimension->name = apr_pstrdup(ctx->pool,name); @@ -134,16 +89,13 @@ dimension->unit = apr_pstrdup(ctx->pool,unit); } - if(skip_validation && !strcmp(skip_validation,"true")) { - dimension->skip_validation = MAPCACHE_TRUE; - } - if(default_value && *default_value) { dimension->default_value = apr_pstrdup(ctx->pool,default_value); } else { ctx->set_error(ctx,400,"dimension \"%s\" has no \"default\" attribute",dimension->name); return; } + dimension->configuration_parse_xml(ctx,dimension,dimension_node); GC_CHECK_ERROR(ctx); @@ -154,7 +106,51 @@ ctx->set_error(ctx, 400, " for tileset \"%s\" has no dimensions defined (expecting children)",tileset->name); return; } + tileset->dimensions = dimensions; + dimension_node = ezxml_child(node,"store_assemblies"); + if(dimension_node && dimension_node->txt) { + if(!strcmp(dimension_node->txt,"false")) { + tileset->store_dimension_assemblies = 0; + } else if(strcmp(dimension_node->txt,"true")) { + ctx->set_error(ctx,400,"failed to parse (%s), expecting \"true\" or \"false\"",dimension_node->txt); + return; + } + } + + dimension_node = ezxml_child(node,"assembly_type"); + if(dimension_node) { + if(!strcmp(dimension_node->txt,"stack")) { + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_STACK; + } else if(!strcmp(dimension_node->txt,"animate")) { + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_ANIMATE; + ctx->set_error(ctx,400,"animate dimension assembly mode not implemented"); + return; + } else if(strcmp(dimension_node->txt,"none")) { + ctx->set_error(ctx,400,"unknown dimension assembly mode (%s). Can be one of \"stack\" or \"none\"",dimension_node->txt); + return; + } else { + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_NONE; + } + } + + /* should we create subdimensions from source if not found in cache. + e.g. if dimension=mosaic returns dimension=val1,val2,val3 should we + query the wms source with dimension=val1 , dimension=val2 and/or + dimension=val3 if they are not found in cache */ + dimension_node = ezxml_child(node,"subdimensions_read_only"); + if(dimension_node) { + if(tileset->dimension_assembly_type == MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + ctx->set_error(ctx,400," used on a tileset with no set, which makes no sense"); + return; + } + if(dimension_node && dimension_node->txt && !strcmp(dimension_node->txt,"true")) { + tileset->subdimension_read_only = 1; + } else if(strcmp(dimension_node->txt,"false")) { + ctx->set_error(ctx,400,"failed to parse (%s), expecting \"true\" or \"false\"",dimension_node->txt); + return; + } + } } void parseGrid(mapcache_context *ctx, ezxml_t node, mapcache_cfg *config) @@ -340,14 +336,14 @@ source = NULL; if(!strcmp(type,"wms")) { source = mapcache_source_wms_create(ctx); -#ifdef USE_MAPSERVER } else if(!strcmp(type,"mapserver")) { source = mapcache_source_mapserver_create(ctx); -#endif } else if(!strcmp(type,"gdal")) { source = mapcache_source_gdal_create(ctx); } else if(!strcmp(type,"dummy")) { source = mapcache_source_dummy_create(ctx); + } else if(!strcmp(type,"fallback")) { + source = mapcache_source_fallback_create(ctx); } else { ctx->set_error(ctx, 400, "unknown source type %s for source \"%s\"", type, name); return; @@ -362,8 +358,22 @@ parseMetadata(ctx, cur_node, source->metadata); GC_CHECK_ERROR(ctx); } + if ((cur_node = ezxml_child(node,"retries")) != NULL) { + source->retry_count = atoi(cur_node->txt); + if(source->retry_count > 10) { + ctx->set_error(ctx,400,"source (%s) count of %d is unreasonably large. max is 10", source->name, source->retry_count); + return; + } + } + if ((cur_node = ezxml_child(node,"retry_delay")) != NULL) { + source->retry_delay = (double)atof(cur_node->txt); + if(source->retry_delay < 0) { + ctx->set_error(ctx,400,"source (%s) retry delay of %f must be positive",source->name, source->retry_delay); + return; + } + } - source->configuration_parse_xml(ctx,node,source); + source->configuration_parse_xml(ctx,node,source, config); GC_CHECK_ERROR(ctx); source->configuration_check(ctx,config,source); GC_CHECK_ERROR(ctx); @@ -422,6 +432,7 @@ } } else if(!strcmp(type,"JPEG")) { int quality = 95; + int optimize = TRUE; mapcache_photometric photometric = MAPCACHE_PHOTOMETRIC_YCBCR; if ((cur_node = ezxml_child(node,"quality")) != NULL) { char *endptr; @@ -445,10 +456,24 @@ return; } } + if ((cur_node = ezxml_child(node,"optimize")) != NULL) { + if(cur_node->txt && !strcasecmp(cur_node->txt,"false")) + optimize = MAPCACHE_OPTIMIZE_NO; + else if(cur_node->txt && !strcasecmp(cur_node->txt,"true")) + optimize = MAPCACHE_OPTIMIZE_YES; + else if(cur_node->txt && !strcasecmp(cur_node->txt,"arithmetic")) + optimize = MAPCACHE_OPTIMIZE_ARITHMETIC; + else { + ctx->set_error(ctx,500,"failed to parse jpeg format %s optimize %s. expecting true, false or arithmetic", + name,cur_node->txt); + return; + } + } format = mapcache_imageio_create_jpeg_format(ctx->pool, - name,quality,photometric); + name,quality,photometric,optimize); } else if(!strcasecmp(type,"MIXED")) { mapcache_image_format *transparent=NULL, *opaque=NULL; + unsigned int alpha_cutoff=255; if ((cur_node = ezxml_child(node,"transparent")) != NULL) { transparent = mapcache_configuration_get_image_format(config,cur_node->txt); } @@ -467,7 +492,10 @@ name,cur_node->txt,cur_node->txt); return; } - format = mapcache_imageio_create_mixed_format(ctx->pool,name,transparent, opaque); + if ((cur_node = ezxml_child(node,"alpha_cutoff")) != NULL) { + alpha_cutoff = atoi(cur_node->txt); + } + format = mapcache_imageio_create_mixed_format(ctx->pool,name,transparent, opaque, alpha_cutoff); } else { ctx->set_error(ctx, 400, "unknown format type %s for format \"%s\"", type, name); return; @@ -486,6 +514,7 @@ { char *name = NULL, *type = NULL; mapcache_cache *cache = NULL; + ezxml_t cur_node; name = (char*)ezxml_attr(node,"name"); type = (char*)ezxml_attr(node,"type"); if(!name || !strlen(name)) { @@ -520,69 +549,48 @@ } else if(!strcmp(type,"google")) { cache = mapcache_cache_google_create(ctx); } else if(!strcmp(type,"bdb")) { -#ifdef USE_BDB cache = mapcache_cache_bdb_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": Berkeley DB support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"tokyocabinet")) { -#ifdef USE_TC cache = mapcache_cache_tc_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": Tokyo Cabinet support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"sqlite3")) { -#ifdef USE_SQLITE cache = mapcache_cache_sqlite_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": sqlite support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"mbtiles")) { -#ifdef USE_SQLITE cache = mapcache_cache_mbtiles_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": sqlite support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"memcache")) { -#ifdef USE_MEMCACHE cache = mapcache_cache_memcache_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": memcache support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"tiff")) { -#ifdef USE_TIFF cache = mapcache_cache_tiff_create(ctx); -#else - ctx->set_error(ctx,400, "failed to add cache \"%s\": tiff support is not available on this build",name); - return; -#endif } else if(!strcmp(type,"couchbase")) { -#ifdef USE_COUCHBASE cache = mapcache_cache_couchbase_create(ctx); -#else - ctx->set_error(ctx, 400, "failed to add cache \"%s\": couchbase support is not available on this build", name); -#endif } else if(!strcmp(type,"riak")) { -#ifdef USE_RIAK cache = mapcache_cache_riak_create(ctx); -#else - ctx->set_error(ctx, 400, "failed to add cache \"%s\": riak support is not available on this build", name); -#endif } else { ctx->set_error(ctx, 400, "unknown cache type %s for cache \"%s\"", type, name); return; } + GC_CHECK_ERROR(ctx); if(cache == NULL) { ctx->set_error(ctx, 400, "failed to parse cache \"%s\"", name); return; } cache->name = name; + if ((cur_node = ezxml_child(node,"retries")) != NULL) { + cache->retry_count = atoi(cur_node->txt); + if(cache->retry_count > 10) { + ctx->set_error(ctx,400,"cache (%s) count of %d is unreasonably large. max is 10", cache->name, cache->retry_count); + return; + } + } + if ((cur_node = ezxml_child(node,"retry_delay")) != NULL) { + cache->retry_delay = (double)atof(cur_node->txt); + if(cache->retry_delay < 0) { + ctx->set_error(ctx,400,"cache (%s) retry delay of %f must be positive",cache->name, cache->retry_delay); + return; + } + } + + cache->configuration_parse_xml(ctx,node,cache,config); GC_CHECK_ERROR(ctx); mapcache_configuration_add_cache(config,cache,name); @@ -824,12 +832,6 @@ GC_CHECK_ERROR(ctx); } - if ((cur_node = ezxml_child(node,"timedimension")) != NULL) { - parseTimeDimension(ctx, cur_node, tileset); - GC_CHECK_ERROR(ctx); - } - - if ((cur_node = ezxml_child(node,"cache")) != NULL) { mapcache_cache *cache = mapcache_configuration_get_cache(config, cur_node->txt); if(!cache) { @@ -1115,8 +1117,6 @@ } else if ((node = ezxml_child(doc,"services")) != NULL) { ctx->log(ctx,MAPCACHE_WARN," tag is deprecated, use "); parseServices(ctx, node, config); - } else { - ctx->set_error(ctx, 400, "no configured"); } if(GC_HAS_ERROR(ctx)) goto cleanup; @@ -1158,31 +1158,8 @@ mapcache_config_parse_locker(ctx,node,&config->locker); GC_CHECK_ERROR(ctx); } else { - /* backwards compatibility */ - int micro_retry; - mapcache_locker_disk *ldisk; - config->locker = mapcache_locker_disk_create(ctx); - ldisk = (mapcache_locker_disk*)config->locker; - if((node = ezxml_child(doc,"lock_dir")) != NULL) { - ldisk->dir = apr_pstrdup(ctx->pool, node->txt); - } else { - ldisk->dir = apr_pstrdup(ctx->pool,"/tmp"); - } - - if((node = ezxml_child(doc,"lock_retry")) != NULL) { - char *endptr; - micro_retry = strtol(node->txt,&endptr,10); - if(*endptr != 0 || micro_retry <= 0) { - ctx->set_error(ctx, 400, "failed to parse lock_retry microseconds \"%s\". Expecting a positive integer", - node->txt); - return; - } - } else { - /* default retry interval is 1/100th of a second, i.e. 10000 microseconds */ - micro_retry = 10000; - } - config->locker->retry_interval = micro_retry / 1000000.0; - config->locker->timeout=120; + mapcache_config_parse_locker_old(ctx,doc,config); + GC_CHECK_ERROR(ctx); } if((node = ezxml_child(doc,"threaded_fetching")) != NULL) { diff -Nru mapcache-1.4.0/lib/connection_pool.c mapcache-1.6.1/lib/connection_pool.c --- mapcache-1.4.0/lib/connection_pool.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/connection_pool.c 2017-09-29 21:31:13.000000000 +0000 @@ -64,7 +64,7 @@ mapcache_pooled_connection *pc = pcc->head; while(pc) { mapcache_pooled_connection *this = pc; - this->private->destructor(this->connection, pcc->pool); + this->private->destructor(this->connection); free(this->private->key); pc = this->private->next; free(this); @@ -73,11 +73,12 @@ return MAPCACHE_SUCCESS; } + apr_status_t mapcache_connection_pool_create(mapcache_connection_pool **cp, apr_pool_t *server_pool) { apr_status_t rv; *cp = apr_pcalloc(server_pool, sizeof(mapcache_connection_pool)); (*cp)->server_pool = server_pool; - rv = apr_reslist_create(&((*cp)->connexions), 1, 5, 200, 60*1000000, + rv = apr_reslist_create(&((*cp)->connexions), 1, 5, 1024, 60*1000000, mapcache_connection_container_creator, mapcache_connection_container_destructor, NULL, @@ -126,7 +127,7 @@ /* ctx->log(ctx, MAPCACHE_DEBUG, "calling constructor for pooled connection (%s)", key); */ - constructor(ctx, &pc->connection, params, pcc->pool); + constructor(ctx, &pc->connection, params); if(GC_HAS_ERROR(ctx)) { free(pc); apr_reslist_release(ctx->connection_pool->connexions, pcc); @@ -150,7 +151,7 @@ count++; } ctx->log(ctx, MAPCACHE_DEBUG, "tearing down pooled connection (%s) to make room", opc->private->key); - opc->private->destructor(opc->connection, pcc->pool); + opc->private->destructor(opc->connection); free(opc->private->key); free(opc->private); free(opc); @@ -172,13 +173,15 @@ } else { pcc->head = pc->private->next; } - pc->private->destructor(pc->connection, pcc->pool); + pc->private->destructor(pc->connection); free(pc->private->key); free(pc); + break; } pred = pc; pc = pc->private->next; } + apr_reslist_release(ctx->connection_pool->connexions,(void*)pcc); } void mapcache_connection_pool_release_connection(mapcache_context *ctx, mapcache_pooled_connection *connection) { diff -Nru mapcache-1.4.0/lib/core.c mapcache-1.6.1/lib/core.c --- mapcache-1.4.0/lib/core.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/core.c 2017-09-29 21:31:13.000000000 +0000 @@ -196,10 +196,13 @@ { int expires = 0; mapcache_http_response *response; - int i,is_empty=1; /* response image is initially empty */; char *timestr; - mapcache_image *base=NULL; - mapcache_image_format *format = NULL; + mapcache_image *base; + mapcache_image_format *format; + mapcache_image_format_type t; + int i,is_empty=1; /* response image is initially empty */; + base=NULL; + format = NULL; #ifdef DEBUG if(req_tile->ntiles ==0) { @@ -208,7 +211,7 @@ } #endif response = mapcache_http_response_create(ctx->pool); - + if(ctx->supports_redirects && req_tile->ntiles == 1) { req_tile->tiles[0]->allow_redirect = 1; } @@ -233,7 +236,7 @@ if(tile->expires && (tile->expires < expires || expires == 0)) { expires = tile->expires; } - + if(tile->nodata) { /* treat the special case where the cache explicitely stated that the tile was empty, and we don't have any vertical merging to do */ @@ -244,8 +247,8 @@ } continue; } - - /* treat the most common case: + + /* treat the most common case: - we have a single tile request (i.e. isempty is true) - the cache returned the encoded image */ @@ -282,7 +285,7 @@ mapcache_image_merge(ctx, base, tile->raw_image); } else { /* we don't need to merge onto an existing tile and don't have access to the tile's encoded data. - * + * * we don't encode the tile's raw image data just yet because we might need to merge another one on top * of it later. */ @@ -307,20 +310,20 @@ return NULL; } } else { + unsigned char empty[5] = {'#',0,0,0,0}; #ifdef DEBUG if(!is_empty) { ctx->set_error(ctx,500,"BUG: no image data to encode, but tile not marked as empty"); return NULL; } #endif - unsigned char empty[5] = {'#',0,0,0,0}; response->data = mapcache_empty_png_decode(ctx,req_tile->tiles[0]->grid_link->grid->tile_sx, req_tile->tiles[0]->grid_link->grid->tile_sy, empty,&is_empty); /* is_empty is unchanged and left to 1 */ format = mapcache_configuration_get_image_format(ctx->config,"PNG8"); } } - + /* compute the content-type */ - mapcache_image_format_type t = mapcache_imageio_header_sniff(ctx,response->data); + t = mapcache_imageio_header_sniff(ctx,response->data); if(t == GC_PNG) apr_table_set(response->headers,"Content-Type","image/png"); else if(t == GC_JPEG) @@ -345,12 +348,13 @@ mapcache_tile ***maptiles; int *nmaptiles; mapcache_tile **tiles; + mapcache_grid_link **effectively_used_grid_links; mapcache_map *basemap = NULL; int ntiles = 0; int i; maptiles = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_tile**)); nmaptiles = apr_pcalloc(ctx->pool,nmaps*sizeof(int)); - mapcache_grid_link **effectively_used_grid_links = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_grid_link*)); + effectively_used_grid_links = apr_pcalloc(ctx->pool,nmaps*sizeof(mapcache_grid_link*)); for(i=0; itileset,maps[i]->grid_link, &maps[i]->extent, maps[i]->width, maps[i]->height, @@ -454,7 +458,7 @@ return NULL; } } - basemap->tileset->source->render_map(ctx, basemap); + mapcache_source_render_map(ctx, basemap->tileset->source, basemap); if(GC_HAS_ERROR(ctx)) return NULL; if(req_map->nmaps>1) { if(!basemap->raw_image) { @@ -463,7 +467,7 @@ } for(i=1; inmaps; i++) { mapcache_map *overlaymap = req_map->maps[i]; - overlaymap->tileset->source->render_map(ctx, overlaymap); + mapcache_source_render_map(ctx, overlaymap->tileset->source, overlaymap); if(GC_HAS_ERROR(ctx)) return NULL; if(!overlaymap->raw_image) { overlaymap->raw_image = mapcache_imageio_decode(ctx,overlaymap->encoded_data); @@ -575,7 +579,7 @@ ctx->set_error(ctx,404, "unsupported feature info format %s",fi->format); return NULL; } - tileset->source->query_info(ctx,fi); + mapcache_source_query_info(ctx, tileset->source, fi); if(GC_HAS_ERROR(ctx)) return NULL; response = mapcache_http_response_create(ctx->pool); response->data = fi->data; @@ -616,7 +620,7 @@ msg = ctx->_errmsg; if(!msg) { - msg = apr_pstrdup(ctx->pool,"an unspecified error has occured"); + msg = apr_pstrdup(ctx->pool,"an unspecified error has occurred"); } ctx->log(ctx,MAPCACHE_ERROR,msg); diff -Nru mapcache-1.4.0/lib/dimension.c mapcache-1.6.1/lib/dimension.c --- mapcache-1.4.0/lib/dimension.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/dimension.c 2017-09-29 21:31:13.000000000 +0000 @@ -37,114 +37,153 @@ #include #include #endif +#ifdef USE_PCRE +#include +#else +#include +#endif + +typedef struct mapcache_dimension_time mapcache_dimension_time; +typedef struct mapcache_dimension_time_sqlite mapcache_dimension_time_sqlite; +typedef struct mapcache_dimension_intervals mapcache_dimension_intervals; +typedef struct mapcache_dimension_values mapcache_dimension_values; +typedef struct mapcache_dimension_sqlite mapcache_dimension_sqlite; +typedef struct mapcache_dimension_regex mapcache_dimension_regex; +typedef struct mapcache_dimension_composite mapcache_dimension_composite; + +struct mapcache_dimension_values { + mapcache_dimension dimension; + apr_array_header_t *values; + int case_sensitive; +}; + +struct mapcache_dimension_sqlite { + mapcache_dimension dimension; + char *dbfile; + char *get_values_for_entry_query; + char *get_all_values_query; +}; + +struct mapcache_dimension_regex { + mapcache_dimension dimension; + char *regex_string; +#ifdef USE_PCRE + pcre *pcregex; +#else + regex_t *regex; +#endif +}; +struct mapcache_dimension_time { + mapcache_dimension_sqlite dimension; +}; -static int _mapcache_dimension_intervals_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +#ifndef HAVE_TIMEGM +time_t timegm(struct tm *tm) { - int i; - char *endptr; - mapcache_dimension_intervals *dimension; - double val = strtod(*value,&endptr); - *value = apr_psprintf(ctx->pool,"%g",val); - if(*endptr != 0) { - return MAPCACHE_FAILURE; - } - dimension = (mapcache_dimension_intervals*)dim; - for(i=0; inintervals; i++) { - double diff; - mapcache_interval *interval = &dimension->intervals[i]; - if(valstart || val>interval->end) - continue; - if(interval->resolution == 0) - return MAPCACHE_SUCCESS; - diff = fmod((val - interval->start),interval->resolution); - if(diff == 0.0) - return MAPCACHE_SUCCESS; - } - return MAPCACHE_FAILURE; + time_t t, tdiff; + struct tm in, gtime, ltime; + + memcpy(&in, tm, sizeof(in)); + t = mktime(&in); + + memcpy(>ime, gmtime(&t), sizeof(gtime)); + memcpy(<ime, localtime(&t), sizeof(ltime)); + gtime.tm_isdst = ltime.tm_isdst; + tdiff = t - mktime(>ime); + + memcpy(&in, tm, sizeof(in)); + return mktime(&in) + tdiff; } +#endif -static apr_array_header_t* _mapcache_dimension_intervals_print(mapcache_context *ctx, mapcache_dimension *dim) -{ - mapcache_dimension_intervals *dimension = (mapcache_dimension_intervals*)dim; - apr_array_header_t *ret = apr_array_make(ctx->pool,dimension->nintervals,sizeof(char*)); - int i; - for(i=0; inintervals; i++) { - mapcache_interval *interval = &dimension->intervals[i]; - APR_ARRAY_PUSH(ret,char*) = apr_psprintf(ctx->pool,"%g/%g/%g",interval->start,interval->end,interval->resolution); +apr_array_header_t *mapcache_requested_dimensions_clone(apr_pool_t *pool, apr_array_header_t *src) { + apr_array_header_t *ret = NULL; + if(src) { + int i; + ret = apr_array_make(pool,src->nelts,sizeof(mapcache_requested_dimension*)); + for(i=0; inelts; i++) { + mapcache_requested_dimension *tiledim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + mapcache_requested_dimension *srcdim = APR_ARRAY_IDX(src,i,mapcache_requested_dimension*); + *tiledim = *srcdim; + APR_ARRAY_PUSH(ret,mapcache_requested_dimension*) = tiledim; + } } return ret; } - -static void _mapcache_dimension_intervals_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, - ezxml_t node) -{ - mapcache_dimension_intervals *dimension; - char *key,*last; - char *values; - const char *entry = node->txt; - int count = 1; - if(!entry || !*entry) { - ctx->set_error(ctx,400,"failed to parse dimension values: none supplied"); +void mapcache_set_requested_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value) { + int i; + if(!dimensions || dimensions->nelts <= 0) { + ctx->set_error(ctx,500,"BUG: no dimensions configure for tile/map"); return; } - dimension = (mapcache_dimension_intervals*)dim; - values = apr_pstrdup(ctx->pool,entry); - - for(key=values; *key; key++) if(*key == ',') count++; - - dimension->intervals = (mapcache_interval*)apr_pcalloc(ctx->pool,count*sizeof(mapcache_interval)); - - for (key = apr_strtok(values, ",", &last); key != NULL; - key = apr_strtok(NULL, ",", &last)) { - char *endptr; - mapcache_interval *interval = &dimension->intervals[dimension->nintervals]; - interval->start = strtod(key,&endptr); - if(*endptr != '/') { - ctx->set_error(ctx,400,"failed to parse min dimension value \"%s\" in \"%s\" for dimension %s",key,entry,dim->name); + for(i=0;inelts;i++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(dimensions,i,mapcache_requested_dimension*); + if(!strcasecmp(dim->dimension->name,name)) { + dim->requested_value = value?apr_pstrdup(ctx->pool,value):NULL; return; } - key = endptr+1; + } + ctx->set_error(ctx,500,"BUG: dimension (%s) not found in tile/map",name); +} - interval->end = strtod(key,&endptr); - if(*endptr != '/') { - ctx->set_error(ctx,400,"failed to parse max dimension value \"%s\" in \"%s\" for dimension %s",key,entry,dim->name); - return; - } - key = endptr+1; - interval->resolution = strtod(key,&endptr); - if(*endptr != '\0') { - ctx->set_error(ctx,400,"failed to parse resolution dimension value \"%s\" in \"%s\" for dimension %s",key,entry,dim->name); +void mapcache_set_cached_dimension(mapcache_context *ctx, apr_array_header_t *dimensions, const char *name, const char *value) { + int i; + if(!dimensions || dimensions->nelts <= 0) { + ctx->set_error(ctx,500,"BUG: no dimensions configure for tile/map"); + return; + } + for(i=0;inelts;i++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(dimensions,i,mapcache_requested_dimension*); + if(!strcasecmp(dim->dimension->name,name)) { + dim->cached_value = value?apr_pstrdup(ctx->pool,value):NULL; return; } - dimension->nintervals++; } + ctx->set_error(ctx,500,"BUG: dimension (%s) not found in tile/map",name); +} - if(!dimension->nintervals) { - ctx->set_error(ctx, 400, " \"%s\" has no intervals",dim->name); - return; - } +void mapcache_tile_set_cached_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value) { + mapcache_set_cached_dimension(ctx,tile->dimensions,name,value); } -static int _mapcache_dimension_regex_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +void mapcache_map_set_cached_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value) { + mapcache_set_cached_dimension(ctx,map->dimensions,name,value); +} +void mapcache_tile_set_requested_dimension(mapcache_context *ctx, mapcache_tile *tile, const char *name, const char *value) { + mapcache_set_requested_dimension(ctx,tile->dimensions,name,value); +} + +void mapcache_map_set_requested_dimension(mapcache_context *ctx, mapcache_map *map, const char *name, const char *value) { + mapcache_set_requested_dimension(ctx,map->dimensions,name,value); +} + +static apr_array_header_t* _mapcache_dimension_regex_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dim, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_regex *dimension = (mapcache_dimension_regex*)dim; + apr_array_header_t *values = apr_array_make(ctx->pool,1,sizeof(char*)); #ifdef USE_PCRE int ovector[30]; - int rc = pcre_exec(dimension->pcregex,NULL,*value,strlen(*value),0,0,ovector,30); - if(rc>0) - return MAPCACHE_SUCCESS; + int rc = pcre_exec(dimension->pcregex,NULL,value,strlen(value),0,0,ovector,30); + if(rc>0) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); + } #else - if(!regexec(dimension->regex,*value,0,0,0)) { - return MAPCACHE_SUCCESS; + if(!regexec(dimension->regex,value,0,0,0)) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); } #endif - return MAPCACHE_FAILURE; + else { + ctx->set_error(ctx,400,"failed to validate requested value for dimension (%s)",dim->name); + } + return values; } -static apr_array_header_t* _mapcache_dimension_regex_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_regex_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_regex *dimension = (mapcache_dimension_regex*)dim; apr_array_header_t *ret = apr_array_make(ctx->pool,1,sizeof(char*)); @@ -157,32 +196,35 @@ ezxml_t node) { mapcache_dimension_regex *dimension; - const char *entry = node->txt; - if(!entry || !*entry) { - ctx->set_error(ctx,400,"failed to parse dimension regex: none supplied"); + ezxml_t child_node = ezxml_child(node,"regex"); + + dimension = (mapcache_dimension_regex*)dim; + + if(child_node && child_node->txt && *child_node->txt) { + dimension->regex_string = apr_pstrdup(ctx->pool,child_node->txt); + } else { + ctx->set_error(ctx,400,"failed to parse dimension regex: no child supplied"); return; } - dimension = (mapcache_dimension_regex*)dim; - dimension->regex_string = apr_pstrdup(ctx->pool,entry); #ifdef USE_PCRE { const char *pcre_err; int pcre_offset; - dimension->pcregex = pcre_compile(entry,0,&pcre_err, &pcre_offset,0); + dimension->pcregex = pcre_compile(dimension->regex_string,0,&pcre_err, &pcre_offset,0); if(!dimension->pcregex) { ctx->set_error(ctx,400,"failed to compile regular expression \"%s\" for dimension \"%s\": %s", - entry,dim->name,pcre_err); + dimension->regex_string,dim->name,pcre_err); return; } } #else { - int rc = regcomp(dimension->regex, entry, REG_EXTENDED); + int rc = regcomp(dimension->regex, dimension->regex_string, REG_EXTENDED); if(rc) { char errmsg[200]; regerror(rc,dimension->regex,errmsg,200); ctx->set_error(ctx,400,"failed to compile regular expression \"%s\" for dimension \"%s\": %s", - entry,dim->name,errmsg); + dimension->regex_string,dim->name,errmsg); return; } } @@ -190,29 +232,40 @@ } -static int _mapcache_dimension_values_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) +static apr_array_header_t* _mapcache_dimension_values_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dim, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { int i; mapcache_dimension_values *dimension = (mapcache_dimension_values*)dim; - for(i=0; invalues; i++) { + apr_array_header_t *values = apr_array_make(ctx->pool,1,sizeof(char*)); + for(i=0; ivalues->nelts; i++) { + char *cur_val = APR_ARRAY_IDX(dimension->values,i,char*); if(dimension->case_sensitive) { - if(!strcmp(*value,dimension->values[i])) - return MAPCACHE_SUCCESS; + if(!strcmp(value,cur_val)) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); + break; + } } else { - if(!strcasecmp(*value,dimension->values[i])) - return MAPCACHE_SUCCESS; + if(!strcasecmp(value,cur_val)) { + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,value); + break; + } } } - return MAPCACHE_FAILURE; + if(i == dimension->values->nelts) { + ctx->set_error(ctx,400,"failed to validate requested value for dimension (%s)",dim->name); + } + return values; } -static apr_array_header_t* _mapcache_dimension_values_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_values_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_values *dimension = (mapcache_dimension_values*)dim; - apr_array_header_t *ret = apr_array_make(ctx->pool,dimension->nvalues,sizeof(char*)); + apr_array_header_t *ret = apr_array_make(ctx->pool,dimension->values->nelts,sizeof(char*)); int i; - for(i=0; invalues; i++) { - APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,dimension->values[i]); + for(i=0; ivalues->nelts; i++) { + APR_ARRAY_PUSH(ret,char*) = apr_pstrdup(ctx->pool,APR_ARRAY_IDX(dimension->values,i,char*)); } return ret; } @@ -221,81 +274,59 @@ static void _mapcache_dimension_values_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, ezxml_t node) { - int count = 1; mapcache_dimension_values *dimension; - const char *case_sensitive; - char *key,*last; - char *values; - const char *entry = node->txt; - if(!entry || !*entry) { - ctx->set_error(ctx,400,"failed to parse dimension values: none supplied"); + ezxml_t child_node = ezxml_child(node,"value"); + dimension = (mapcache_dimension_values*)dim; + + if(!child_node) { + ctx->set_error(ctx,400,"failed to parse dimension values: no children supplied"); return; } - - dimension = (mapcache_dimension_values*)dim; - case_sensitive = ezxml_attr(node,"case_sensitive"); - if(case_sensitive && !strcasecmp(case_sensitive,"true")) { - dimension->case_sensitive = 1; + for(; child_node; child_node = child_node->next) { + const char* entry = child_node->txt; + if(!entry || !*entry) { + ctx->set_error(ctx,400,"failed to parse dimension values: empty "); + return; + } + APR_ARRAY_PUSH(dimension->values,char*) = apr_pstrdup(ctx->pool,entry); } - values = apr_pstrdup(ctx->pool,entry); - for(key=values; *key; key++) if(*key == ',') count++; - - dimension->values = (char**)apr_pcalloc(ctx->pool,count*sizeof(char*)); - - for (key = apr_strtok(values, ",", &last); key != NULL; - key = apr_strtok(NULL, ",", &last)) { - dimension->values[dimension->nvalues]=key; - dimension->nvalues++; + child_node = ezxml_child(node,"case_sensitive"); + if(child_node && child_node->txt) { + if(!strcasecmp(child_node->txt,"true")) { + dimension->case_sensitive = 1; + } } - if(!dimension->nvalues) { + + if(!dimension->values->nelts) { ctx->set_error(ctx, 400, " \"%s\" has no values",dim->name); return; } } -mapcache_dimension* mapcache_dimension_values_create(apr_pool_t *pool) +mapcache_dimension* mapcache_dimension_values_create(mapcache_context *ctx, apr_pool_t *pool) { mapcache_dimension_values *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_values)); dimension->dimension.type = MAPCACHE_DIMENSION_VALUES; - dimension->nvalues = 0; - dimension->dimension.validate = _mapcache_dimension_values_validate; + dimension->values = apr_array_make(pool,1,sizeof(char*)); + dimension->dimension.get_entries_for_value = _mapcache_dimension_values_get_entries_for_value; dimension->dimension.configuration_parse_xml = _mapcache_dimension_values_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_values_print; - return (mapcache_dimension*)dimension; -} - -mapcache_dimension* mapcache_dimension_time_create(apr_pool_t *pool) -{ - mapcache_dimension_time *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_time)); - dimension->dimension.type = MAPCACHE_DIMENSION_TIME; - dimension->nintervals = 0; -// dimension->dimension.validate = _mapcache_dimension_time_validate; -// dimension->dimension.parse = _mapcache_dimension_time_parse; -// dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_time_print; + dimension->dimension.get_all_entries = _mapcache_dimension_values_get_all_entries; + dimension->dimension.get_all_ogc_formatted_entries = _mapcache_dimension_values_get_all_entries; return (mapcache_dimension*)dimension; } -mapcache_dimension* mapcache_dimension_intervals_create(apr_pool_t *pool) -{ - mapcache_dimension_intervals *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_intervals)); - dimension->dimension.type = MAPCACHE_DIMENSION_INTERVALS; - dimension->nintervals = 0; - dimension->dimension.validate = _mapcache_dimension_intervals_validate; - dimension->dimension.configuration_parse_xml = _mapcache_dimension_intervals_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_intervals_print; - return (mapcache_dimension*)dimension; -} -mapcache_dimension* mapcache_dimension_regex_create(apr_pool_t *pool) +mapcache_dimension* mapcache_dimension_regex_create(mapcache_context *ctx, apr_pool_t *pool) { mapcache_dimension_regex *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_regex)); dimension->dimension.type = MAPCACHE_DIMENSION_REGEX; #ifndef USE_PCRE dimension->regex = (regex_t*)apr_pcalloc(pool, sizeof(regex_t)); #endif - dimension->dimension.validate = _mapcache_dimension_regex_validate; + dimension->dimension.get_entries_for_value = _mapcache_dimension_regex_get_entries_for_value; dimension->dimension.configuration_parse_xml = _mapcache_dimension_regex_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_regex_print; + dimension->dimension.get_all_entries = _mapcache_dimension_regex_get_all_entries; + dimension->dimension.get_all_ogc_formatted_entries = _mapcache_dimension_regex_get_all_entries; return (mapcache_dimension*)dimension; } @@ -307,16 +338,16 @@ int n_statements; }; -void mapcache_sqlite_dimension_connection_constructor(mapcache_context *ctx, void **conn_, void *params, apr_pool_t *pool) +void mapcache_sqlite_dimension_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { int ret; - int flags; + int flags; char *dbfile = (char*) params; struct sqlite_dimension_conn *conn = calloc(1, sizeof (struct sqlite_dimension_conn)); *conn_ = conn; flags = SQLITE_OPEN_READONLY | SQLITE_OPEN_NOMUTEX; ret = sqlite3_open_v2(dbfile, &conn->handle, flags, NULL); - + if (ret != SQLITE_OK) { ctx->set_error(ctx,500,"failed to open sqlite dimension dbfile (%s): %s",dbfile,sqlite3_errmsg(conn->handle)); sqlite3_close(conn->handle); @@ -326,7 +357,7 @@ sqlite3_busy_timeout(conn->handle, 300000); } -void mapcache_sqlite_dimension_connection_destructor(void *conn_, apr_pool_t *pool) +void mapcache_sqlite_dimension_connection_destructor(void *conn_) { struct sqlite_dimension_conn *conn = (struct sqlite_dimension_conn*) conn_; while(conn->n_statements) { @@ -340,14 +371,10 @@ free(conn); } -static mapcache_pooled_connection* _sqlite_time_dimension_get_conn(mapcache_context *ctx, mapcache_timedimension_sqlite *dim) { - mapcache_pooled_connection *pc = mapcache_connection_pool_get_connection(ctx,dim->timedimension.key, - mapcache_sqlite_dimension_connection_constructor, - mapcache_sqlite_dimension_connection_destructor, dim->dbfile); - return pc; -} -static mapcache_pooled_connection* _sqlite_dimension_get_conn(mapcache_context *ctx, mapcache_dimension_sqlite *dim) { - mapcache_pooled_connection *pc = mapcache_connection_pool_get_connection(ctx,dim->dimension.name, +static mapcache_pooled_connection* _sqlite_dimension_get_conn(mapcache_context *ctx, mapcache_tileset *tileset, mapcache_dimension_sqlite *dim) { + mapcache_dimension *pdim = (mapcache_dimension*)dim; + char *conn_key = apr_pstrcat(ctx->pool,"dim_",tileset?tileset->name:"","_",pdim->name,NULL); + mapcache_pooled_connection *pc = mapcache_connection_pool_get_connection(ctx,conn_key, mapcache_sqlite_dimension_connection_constructor, mapcache_sqlite_dimension_connection_destructor, dim->dbfile); return pc; @@ -362,16 +389,85 @@ } } +static void _mapcache_dimension_sqlite_bind_parameters(mapcache_context *ctx, sqlite3_stmt *stmt, sqlite3 *handle, + const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { + int paramidx,ret; + paramidx = sqlite3_bind_parameter_index(stmt, ":dim"); + if (paramidx) { + ret = sqlite3_bind_text(stmt, paramidx, value, -1, SQLITE_STATIC); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "sqlite dimension failed to bind :dim : %s", sqlite3_errmsg(handle)); + return; + } + } + + if(tileset) { + paramidx = sqlite3_bind_parameter_index(stmt, ":tileset"); + if (paramidx) { + ret = sqlite3_bind_text(stmt, paramidx, tileset->name, -1, SQLITE_STATIC); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "sqlite dimension failed to bind :tileset : %s", sqlite3_errmsg(handle)); + return; + } + } + } + + if(grid) { + paramidx = sqlite3_bind_parameter_index(stmt, ":gridsrs"); + if (paramidx) { + ret = sqlite3_bind_text(stmt, paramidx, grid->srs, -1, SQLITE_STATIC); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :gridsrs %s", sqlite3_errmsg(handle)); + return; + } + } + } -static int _mapcache_dimension_sqlite_validate(mapcache_context *ctx, mapcache_dimension *dim, char **value) + paramidx = sqlite3_bind_parameter_index(stmt, ":minx"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->minx:-DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :minx %s", sqlite3_errmsg(handle)); + return; + } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":miny"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->miny:-DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :miny %s", sqlite3_errmsg(handle)); + return; + } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":maxx"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxx:DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :maxx %s", sqlite3_errmsg(handle)); + return; + } + } + paramidx = sqlite3_bind_parameter_index(stmt, ":maxy"); + if (paramidx) { + ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxy:DBL_MAX); + if(ret != SQLITE_OK) { + ctx->set_error(ctx,400, "failed to bind :maxy %s", sqlite3_errmsg(handle)); + return; + } + } +} +static apr_array_header_t* _mapcache_dimension_sqlite_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dim, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_sqlite *dimension = (mapcache_dimension_sqlite*)dim; struct sqlite_dimension_conn *conn = NULL; - int sqliteret,paramidx,ret=MAPCACHE_FAILURE; mapcache_pooled_connection *pc; - pc = _sqlite_dimension_get_conn(ctx,dimension); + apr_array_header_t *values = apr_array_make(ctx->pool,1,sizeof(char*)); + int sqliteret; + pc = _sqlite_dimension_get_conn(ctx,tileset,dimension); if (GC_HAS_ERROR(ctx)) { - return ret; + return values; } conn = pc->connection; if(!conn->prepared_statements) { @@ -379,21 +475,19 @@ conn->n_statements = 2; } if(!conn->prepared_statements[0]) { - sqliteret = sqlite3_prepare_v2(conn->handle, dimension->validate_query, -1, &conn->prepared_statements[0], NULL); - if(sqliteret != SQLITE_OK) { + if(SQLITE_OK != sqlite3_prepare_v2(conn->handle, dimension->get_values_for_entry_query, -1, &conn->prepared_statements[0], NULL)) { ctx->set_error(ctx, 500, "sqlite dimension backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); goto cleanup; } } - - paramidx = sqlite3_bind_parameter_index(conn->prepared_statements[0], ":dim"); - if (paramidx) { - sqliteret = sqlite3_bind_text(conn->prepared_statements[0], paramidx, *value, -1, SQLITE_STATIC); - if(sqliteret != SQLITE_OK) { - ctx->set_error(ctx,400, "sqlite dimension failed to bind :dim : %s", sqlite3_errmsg(conn->handle)); - goto cleanup; - } + + _mapcache_dimension_sqlite_bind_parameters(ctx,conn->prepared_statements[0],conn->handle,value, + tileset,extent,grid); + if (GC_HAS_ERROR(ctx)) { + return values; } + + do { sqliteret = sqlite3_step(conn->prepared_statements[0]); if (sqliteret != SQLITE_DONE && sqliteret != SQLITE_ROW && sqliteret != SQLITE_BUSY && sqliteret != SQLITE_LOCKED) { @@ -401,11 +495,8 @@ goto cleanup; } if(sqliteret == SQLITE_ROW) { - const char* dim_modified = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); - if(strcmp(dim_modified, *value)) { - *value = apr_pstrdup(ctx->pool, dim_modified); - } - ret = MAPCACHE_SUCCESS; + const char* dimrow = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); + APR_ARRAY_PUSH(values,char*) = apr_pstrdup(ctx->pool,dimrow); } } while (sqliteret == SQLITE_ROW || sqliteret == SQLITE_BUSY || sqliteret == SQLITE_LOCKED); @@ -414,18 +505,19 @@ sqlite3_reset(conn->prepared_statements[0]); } _sqlite_dimension_release_conn(ctx,pc); - - return ret; + + return values; } -static apr_array_header_t* _mapcache_dimension_sqlite_print(mapcache_context *ctx, mapcache_dimension *dim) +static apr_array_header_t* _mapcache_dimension_sqlite_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { mapcache_dimension_sqlite *dimension = (mapcache_dimension_sqlite*)dim; struct sqlite_dimension_conn *conn = NULL; int sqliteret; apr_array_header_t *ret = apr_array_make(ctx->pool,0,sizeof(char*)); mapcache_pooled_connection *pc; - pc = _sqlite_dimension_get_conn(ctx,dimension); + pc = _sqlite_dimension_get_conn(ctx,tileset,dimension); if (GC_HAS_ERROR(ctx)) { return ret; } @@ -434,14 +526,18 @@ conn->prepared_statements = calloc(2,sizeof(sqlite3_stmt*)); conn->n_statements = 2; } - + if(!conn->prepared_statements[1]) { - sqliteret = sqlite3_prepare_v2(conn->handle, dimension->list_query, -1, &conn->prepared_statements[1], NULL); + sqliteret = sqlite3_prepare_v2(conn->handle, dimension->get_all_values_query, -1, &conn->prepared_statements[1], NULL); if(sqliteret != SQLITE_OK) { ctx->set_error(ctx, 500, "sqlite dimension backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); goto cleanup; } } + _mapcache_dimension_sqlite_bind_parameters(ctx,conn->prepared_statements[1],conn->handle,NULL,tileset,extent,grid); + if (GC_HAS_ERROR(ctx)) { + return ret; + } do { sqliteret = sqlite3_step(conn->prepared_statements[1]); if (sqliteret != SQLITE_DONE && sqliteret != SQLITE_ROW && sqliteret != SQLITE_BUSY && sqliteret != SQLITE_LOCKED) { @@ -459,7 +555,7 @@ sqlite3_reset(conn->prepared_statements[1]); } _sqlite_dimension_release_conn(ctx,pc); - + return ret; } @@ -469,7 +565,7 @@ { mapcache_dimension_sqlite *dimension; ezxml_t child; - + dimension = (mapcache_dimension_sqlite*)dim; child = ezxml_child(node,"dbfile"); @@ -481,80 +577,30 @@ } child = ezxml_child(node,"validate_query"); if(child) { - dimension->validate_query = apr_pstrdup(ctx->pool, child->txt); + dimension->get_values_for_entry_query = apr_pstrdup(ctx->pool, child->txt); } else { ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); return; } child = ezxml_child(node,"list_query"); if(child) { - dimension->list_query = apr_pstrdup(ctx->pool, child->txt); + dimension->get_all_values_query = apr_pstrdup(ctx->pool, child->txt); } else { ctx->set_error(ctx,400,"sqlite dimension \"%s\" has no node", dim->name); return; } - + } -static void _bind_sqlite_timedimension_params(mapcache_context *ctx, sqlite3_stmt *stmt, - sqlite3 *handle, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, +static void _bind_sqlite_dimension_time_params(mapcache_context *ctx, sqlite3_stmt *stmt, + sqlite3 *handle, const char *dim_value, mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end) { int paramidx,ret; - paramidx = sqlite3_bind_parameter_index(stmt, ":tileset"); - if (paramidx) { - ret = sqlite3_bind_text(stmt, paramidx, tileset->name, -1, SQLITE_STATIC); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :tileset: %s", sqlite3_errmsg(handle)); - return; - } - } - - if(grid) { - paramidx = sqlite3_bind_parameter_index(stmt, ":gridsrs"); - if (paramidx) { - ret = sqlite3_bind_text(stmt, paramidx, grid->srs, -1, SQLITE_STATIC); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :gridsrs %s", sqlite3_errmsg(handle)); - return; - } - } - } - - paramidx = sqlite3_bind_parameter_index(stmt, ":minx"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent?extent->minx:-DBL_MAX); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :minx %s", sqlite3_errmsg(handle)); - return; - } - } - paramidx = sqlite3_bind_parameter_index(stmt, ":miny"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent?extent->miny:-DBL_MAX); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :miny %s", sqlite3_errmsg(handle)); - return; - } - } - paramidx = sqlite3_bind_parameter_index(stmt, ":maxx"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxx:DBL_MAX); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :maxx %s", sqlite3_errmsg(handle)); - return; - } - } - paramidx = sqlite3_bind_parameter_index(stmt, ":maxy"); - if (paramidx) { - ret = sqlite3_bind_double(stmt, paramidx, extent?extent->maxy:DBL_MAX); - if(ret != SQLITE_OK) { - ctx->set_error(ctx,400, "failed to bind :maxy %s", sqlite3_errmsg(handle)); - return; - } - } + _mapcache_dimension_sqlite_bind_parameters(ctx,stmt,handle,dim_value,tileset,extent,grid); + paramidx = sqlite3_bind_parameter_index(stmt, ":start_timestamp"); if (paramidx) { ret = sqlite3_bind_int64(stmt, paramidx, start); @@ -574,14 +620,14 @@ } } -apr_array_header_t *_mapcache_timedimension_sqlite_get_entries(mapcache_context *ctx, mapcache_timedimension *dim, - mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, time_t start, time_t end) { - mapcache_timedimension_sqlite *sdim = (mapcache_timedimension_sqlite*)dim; - int ret; +apr_array_header_t *_mapcache_dimension_time_get_entries(mapcache_context *ctx, mapcache_dimension_time *dim, const char *dim_value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid, time_t *intervals, int n_intervals) { + mapcache_dimension_sqlite *sdim = (mapcache_dimension_sqlite*)dim; + int i,ret; apr_array_header_t *time_ids = NULL; mapcache_pooled_connection *pc; struct sqlite_dimension_conn *conn; - pc = _sqlite_time_dimension_get_conn(ctx,sdim); + pc = _sqlite_dimension_get_conn(ctx,tileset,sdim); if (GC_HAS_ERROR(ctx)) { return NULL; } @@ -591,7 +637,7 @@ conn->n_statements = 1; } if(!conn->prepared_statements[0]) { - ret = sqlite3_prepare_v2(conn->handle, sdim->query, -1, &conn->prepared_statements[0], NULL); + ret = sqlite3_prepare_v2(conn->handle, sdim->get_values_for_entry_query, -1, &conn->prepared_statements[0], NULL); if(ret != SQLITE_OK) { ctx->set_error(ctx, 500, "time sqlite backend failed on preparing query: %s", sqlite3_errmsg(conn->handle)); _sqlite_dimension_release_conn(ctx, pc); @@ -599,33 +645,38 @@ } } - _bind_sqlite_timedimension_params(ctx,conn->prepared_statements[0],conn->handle,tileset,grid,extent,start,end); - if(GC_HAS_ERROR(ctx)) { - _sqlite_dimension_release_conn(ctx, pc); - return NULL; + for(i=0;iprepared_statements[0],conn->handle,dim_value,tileset,grid,extent,intervals[i*2],intervals[i*2+1]); + if(GC_HAS_ERROR(ctx)) { + _sqlite_dimension_release_conn(ctx, pc); + return NULL; + } + + time_ids = apr_array_make(ctx->pool,0,sizeof(char*)); + do { + ret = sqlite3_step(conn->prepared_statements[0]); + if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { + ctx->set_error(ctx, 500, "sqlite backend failed on dimension_time query : %s (%d)", sqlite3_errmsg(conn->handle), ret); + _sqlite_dimension_release_conn(ctx, pc); + return NULL; + } + if(ret == SQLITE_ROW) { + const char* time_id = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); + APR_ARRAY_PUSH(time_ids,char*) = apr_pstrdup(ctx->pool,time_id); + } + } while (ret == SQLITE_ROW || ret == SQLITE_BUSY || ret == SQLITE_LOCKED); + sqlite3_reset(conn->prepared_statements[0]); } - - time_ids = apr_array_make(ctx->pool,0,sizeof(char*)); - do { - ret = sqlite3_step(conn->prepared_statements[0]); - if (ret != SQLITE_DONE && ret != SQLITE_ROW && ret != SQLITE_BUSY && ret != SQLITE_LOCKED) { - ctx->set_error(ctx, 500, "sqlite backend failed on timedimension query : %s (%d)", sqlite3_errmsg(conn->handle), ret); - _sqlite_dimension_release_conn(ctx, pc); - return NULL; - } - if(ret == SQLITE_ROW) { - const char* time_id = (const char*) sqlite3_column_text(conn->prepared_statements[0], 0); - APR_ARRAY_PUSH(time_ids,char*) = apr_pstrdup(ctx->pool,time_id); - } - } while (ret == SQLITE_ROW || ret == SQLITE_BUSY || ret == SQLITE_LOCKED); - sqlite3_reset(conn->prepared_statements[0]); _sqlite_dimension_release_conn(ctx, pc); return time_ids; } -apr_array_header_t *_mapcache_timedimension_sqlite_get_all_entries(mapcache_context *ctx, mapcache_timedimension *dim, - mapcache_tileset *tileset) { - return _mapcache_timedimension_sqlite_get_entries(ctx,dim,tileset,NULL,NULL,0,INT_MAX); + +apr_array_header_t* _mapcache_dimension_time_get_all_entries(mapcache_context *ctx, mapcache_dimension *dim, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { + mapcache_dimension_time *tdim = (mapcache_dimension_time*)dim; + time_t all[2] = {0,INT_MAX}; + return _mapcache_dimension_time_get_entries(ctx,tdim,NULL,tileset,extent,grid,all,1); } #endif @@ -640,30 +691,31 @@ #ifdef USE_SQLITE -void _mapcache_timedimension_sqlite_parse_xml(mapcache_context *ctx, mapcache_timedimension *dim, ezxml_t node) { - mapcache_timedimension_sqlite *sdim = (mapcache_timedimension_sqlite*)dim; +void _mapcache_dimension_time_parse_xml(mapcache_context *ctx, mapcache_dimension *dim, ezxml_t node) { + mapcache_dimension_sqlite *sdim = (mapcache_dimension_sqlite*)dim; + mapcache_dimension *pdim = (mapcache_dimension*)dim; ezxml_t child; - + child = ezxml_child(node,"dbfile"); if(child && child->txt && *child->txt) { sdim->dbfile = apr_pstrdup(ctx->pool,child->txt); } else { - ctx->set_error(ctx,400,"no entry for %s",dim->key); + ctx->set_error(ctx,400,"no entry for %s",pdim->name); return; } child = ezxml_child(node,"query"); if(child && child->txt && *child->txt) { - sdim->query = apr_pstrdup(ctx->pool,child->txt); + sdim->get_values_for_entry_query = apr_pstrdup(ctx->pool,child->txt); } else { - ctx->set_error(ctx,400,"no entry for %s",dim->key); + ctx->set_error(ctx,400,"no entry for %s",pdim->name); return; } } #endif char *mapcache_ogc_strptime(const char *value, struct tm *ts, mapcache_time_interval_t *ti) { - memset (ts, '\0', sizeof (*ts)); char *valueptr; + memset (ts, '\0', sizeof (*ts)); valueptr = strptime(value,"%Y-%m-%dT%H:%M:%SZ",ts); *ti = MAPCACHE_TINTERVAL_SECOND; if(valueptr) return valueptr; @@ -685,97 +737,122 @@ return NULL; } -apr_array_header_t* mapcache_timedimension_get_entries_for_value(mapcache_context *ctx, mapcache_timedimension *timedimension, - mapcache_tileset *tileset, mapcache_grid *grid, mapcache_extent *extent, const char *value) { +#ifdef USE_SQLITE +apr_array_header_t* _mapcache_dimension_time_get_entries_for_value(mapcache_context *ctx, mapcache_dimension *dimension, const char *value, + mapcache_tileset *tileset, mapcache_extent *extent, mapcache_grid *grid) { + /* look if supplied value is a predefined key */ /* split multiple values, loop */ /* extract start and end values */ struct tm tm_start,tm_end; - time_t start,end; + time_t *intervals; mapcache_time_interval_t tis,tie; - char *valueptr = (char*)value; - valueptr = mapcache_ogc_strptime(value,&tm_start,&tis); - if(!valueptr) { - ctx->set_error(ctx,400,"failed to parse time %s",value); - return NULL; - } + char *valueptr = apr_pstrdup(ctx->pool,value); + char *last,*key; + int count=0; + mapcache_dimension_time *dimension_time = (mapcache_dimension_time*)dimension; - if(*valueptr == '/' || (*valueptr == '-' && *(valueptr+1) == '-')) { - /* we have a second (end) time */ - if (*valueptr == '/') { - valueptr++; - } - else { - valueptr += 2; - } - valueptr = mapcache_ogc_strptime(valueptr,&tm_end,&tie); + /*count how many time entries were supplied*/ + for(; *value; value++) if(*value == ',') count++; + + intervals = apr_pcalloc(ctx->pool,2*count*sizeof(time_t)); + count = 0; + + + /* Split the input on '&' */ + for (key = apr_strtok(valueptr, ",", &last); key != NULL; + key = apr_strtok(NULL, ",", &last)) { + valueptr = mapcache_ogc_strptime(key,&tm_start,&tis); if(!valueptr) { - ctx->set_error(ctx,400,"failed to parse end time in %s",value); + ctx->set_error(ctx,400,"failed to parse time %s",value); return NULL; } - } else if(*valueptr == 0) { - tie = tis; - tm_end = tm_start; - } else { - ctx->set_error(ctx,400,"failed (2) to parse time %s",value); - return NULL; - } - end = timegm(&tm_end); - start = timegm(&tm_start); - if(difftime(start,end) == 0) { - switch(tie) { - case MAPCACHE_TINTERVAL_SECOND: - tm_end.tm_sec += 1; - break; - case MAPCACHE_TINTERVAL_MINUTE: - tm_end.tm_min += 1; - break; - case MAPCACHE_TINTERVAL_HOUR: - tm_end.tm_hour += 1; - break; - case MAPCACHE_TINTERVAL_DAY: - tm_end.tm_mday += 1; - break; - case MAPCACHE_TINTERVAL_MONTH: - tm_end.tm_mon += 1; - break; - case MAPCACHE_TINTERVAL_YEAR: - tm_end.tm_year += 1; - break; + + if(*valueptr == '/' || (*valueptr == '-' && *(valueptr+1) == '-')) { + /* we have a second (end) time */ + if (*valueptr == '/') { + valueptr++; + } + else { + valueptr += 2; + } + valueptr = mapcache_ogc_strptime(valueptr,&tm_end,&tie); + if(!valueptr) { + ctx->set_error(ctx,400,"failed to parse end time in %s",value); + return NULL; + } + } else if(*valueptr == 0) { + tie = tis; + tm_end = tm_start; + } else { + ctx->set_error(ctx,400,"failed (2) to parse time %s",value); + return NULL; + } + intervals[count*2+1] = timegm(&tm_end); + intervals[count*2] = timegm(&tm_start); + if(difftime(intervals[count*2],intervals[count*2+1]) == 0) { + switch(tie) { + case MAPCACHE_TINTERVAL_SECOND: + tm_end.tm_sec += 1; + break; + case MAPCACHE_TINTERVAL_MINUTE: + tm_end.tm_min += 1; + break; + case MAPCACHE_TINTERVAL_HOUR: + tm_end.tm_hour += 1; + break; + case MAPCACHE_TINTERVAL_DAY: + tm_end.tm_mday += 1; + break; + case MAPCACHE_TINTERVAL_MONTH: + tm_end.tm_mon += 1; + break; + case MAPCACHE_TINTERVAL_YEAR: + tm_end.tm_year += 1; + break; + } + intervals[count*2+1] = timegm(&tm_end); } - end = timegm(&tm_end); + count++; } - - return timedimension->get_entries_for_interval(ctx,timedimension,tileset,grid,extent,start,end); + return _mapcache_dimension_time_get_entries(ctx,dimension_time,value,tileset,extent,grid,intervals,count); /* end loop */ } +#endif -mapcache_dimension* mapcache_dimension_sqlite_create(apr_pool_t *pool) +mapcache_dimension* mapcache_dimension_sqlite_create(mapcache_context *ctx, apr_pool_t *pool) { #ifdef USE_SQLITE mapcache_dimension_sqlite *dimension = apr_pcalloc(pool, sizeof(mapcache_dimension_sqlite)); dimension->dimension.type = MAPCACHE_DIMENSION_SQLITE; dimension->dbfile = NULL; - dimension->dimension.validate = _mapcache_dimension_sqlite_validate; + dimension->dimension.get_entries_for_value = _mapcache_dimension_sqlite_get_entries_for_value; dimension->dimension.configuration_parse_xml = _mapcache_dimension_sqlite_parse_xml; - dimension->dimension.print_ogc_formatted_values = _mapcache_dimension_sqlite_print; + dimension->dimension.get_all_entries = _mapcache_dimension_sqlite_get_all_entries; + dimension->dimension.get_all_ogc_formatted_entries = _mapcache_dimension_sqlite_get_all_entries; return (mapcache_dimension*)dimension; #else + ctx->set_error(ctx,400,"Sqlite dimension support requires SQLITE support to be built in"); return NULL; #endif } +mapcache_dimension* mapcache_dimension_time_create(mapcache_context *ctx, apr_pool_t *pool) { #ifdef USE_SQLITE -mapcache_timedimension* mapcache_timedimension_sqlite_create(apr_pool_t *pool) { - mapcache_timedimension_sqlite *dim = apr_pcalloc(pool, sizeof(mapcache_timedimension_sqlite)); - dim->timedimension.assembly_type = MAPCACHE_TIMEDIMENSION_ASSEMBLY_STACK; - dim->timedimension.get_entries_for_interval = _mapcache_timedimension_sqlite_get_entries; - dim->timedimension.get_all_entries = _mapcache_timedimension_sqlite_get_all_entries; - dim->timedimension.configuration_parse_xml = _mapcache_timedimension_sqlite_parse_xml; - return (mapcache_timedimension*)dim; -} + mapcache_dimension_time *dim = apr_pcalloc(pool, sizeof(mapcache_dimension_time)); + mapcache_dimension *pdim = (mapcache_dimension*)dim; + pdim->get_entries_for_value = _mapcache_dimension_time_get_entries_for_value; + pdim->get_all_entries = _mapcache_dimension_time_get_all_entries; + pdim->get_all_ogc_formatted_entries = _mapcache_dimension_time_get_all_entries; + pdim->configuration_parse_xml = _mapcache_dimension_time_parse_xml; + return pdim; +#else + ctx->set_error(ctx,400,"TIME dimension support requires SQLITE support to be built in"); + return NULL; #endif +} + /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.4.0/lib/ezxml.c mapcache-1.6.1/lib/ezxml.c --- mapcache-1.4.0/lib/ezxml.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/ezxml.c 2017-09-29 21:31:13.000000000 +0000 @@ -370,7 +370,7 @@ else *s = '\0'; // null terminate tag name for (i = 0; root->attr[i] && strcmp(n, root->attr[i][0]); i++); - while (*(n = ++s + strspn(s, EZXML_WS)) && *n != '>') { + while (*(n = s + 1 + strspn(s + 1, EZXML_WS)) && *n != '>') { if (*(s = n + strcspn(n, EZXML_WS))) *s = '\0'; // attr name else { ezxml_err(root, t, "malformed levels[z]->resolution; @@ -71,6 +71,65 @@ } } +void mapcache_grid_get_metatile_extent(mapcache_context *ctx, mapcache_tile *tile, mapcache_extent *extent) { + mapcache_grid *grid = tile->grid_link->grid; + double res = grid->levels[tile->z]->resolution; + double gbuffer,gwidth,gheight,fullgwidth,fullgheight; + mapcache_tileset *tileset = tile->tileset; + int mtx,mty,blx,bly,mtsx,mtsy; + mtx = tile->x / tileset->metasize_x; + if(tile->x < 0) + mtx --; + mty = tile->y / tileset->metasize_y; + if(tile->y < 0) + mty --; + blx = mtx * tileset->metasize_x; + bly = mty * tileset->metasize_y; + + /* adjust the size of the the metatile so it does not extend past the grid limits. + * If we don't do this, we end up with cut labels on the edges of the tile grid + */ + if(blx+tileset->metasize_x-1 >= grid->levels[tile->z]->maxx) { + mtsx = grid->levels[tile->z]->maxx - blx; + } else { + mtsx = tileset->metasize_x; + } + if(bly+tileset->metasize_y-1 >= grid->levels[tile->z]->maxy) { + mtsy = grid->levels[tile->z]->maxy - bly; + } else { + mtsy = tileset->metasize_y; + } + + /* buffer in geographical units */ + gbuffer = res * tileset->metabuffer; + + /* adjusted metatile size in geographical units */ + gwidth = res * mtsx * grid->tile_sx; + gheight = res * mtsy * grid->tile_sy; + + /* configured metatile size in geographical units */ + fullgwidth = res * tileset->metasize_x * grid->tile_sx; + fullgheight = res * tileset->metasize_y * grid->tile_sy; + + switch(grid->origin) { + case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: + extent->minx = grid->extent.minx + mtx * fullgwidth - gbuffer; + extent->miny = grid->extent.miny + mty * fullgheight - gbuffer; + extent->maxx = extent->minx + gwidth + 2 * gbuffer; + extent->maxy = extent->miny + gheight + 2 * gbuffer; + break; + case MAPCACHE_GRID_ORIGIN_TOP_LEFT: + extent->minx = grid->extent.minx + mtx * fullgwidth - gbuffer; + extent->maxy = grid->extent.maxy - mty * fullgheight + gbuffer; + extent->maxx = extent->minx + gwidth + 2 * gbuffer; + extent->miny = extent->maxy - gheight - 2 * gbuffer; + break; + case MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT: + case MAPCACHE_GRID_ORIGIN_TOP_RIGHT: + ctx->set_error(ctx,500,"origin not implemented"); + } +} + const char* mapcache_grid_get_crs(mapcache_context *ctx, mapcache_grid *grid) { char *epsgnum; diff -Nru mapcache-1.4.0/lib/http.c mapcache-1.6.1/lib/http.c --- mapcache-1.4.0/lib/http.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/http.c 2017-09-29 21:31:13.000000000 +0000 @@ -105,6 +105,7 @@ char error_msg[CURL_ERROR_SIZE]; int ret; struct curl_slist *curl_headers=NULL; + struct _header_struct h; curl_handle = curl_easy_init(); @@ -121,7 +122,6 @@ if(headers != NULL) { /* intercept headers */ - struct _header_struct h; h.headers = headers; h.ctx=ctx; curl_easy_setopt(curl_handle, CURLOPT_HEADERFUNCTION, _mapcache_curl_header_callback); @@ -142,7 +142,7 @@ int i; for (i = 0; i < array->nelts; i++) { char *val = elts[i].val; - if(strchr(val,'{') && ctx->headers_in) { + if(val && strchr(val,'{') && ctx->headers_in) { _header_replace_str(ctx,ctx->headers_in,&val); } curl_headers = curl_slist_append(curl_headers, apr_pstrcat(ctx->pool,elts[i].key,": ",val,NULL)); @@ -156,12 +156,14 @@ if(req->post_body && req->post_len>0) { curl_easy_setopt(curl_handle, CURLOPT_POSTFIELDS, req->post_body); } + + if(!http_code) + curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1); + /* get it! */ ret = curl_easy_perform(curl_handle); if(http_code) curl_easy_getinfo (curl_handle, CURLINFO_RESPONSE_CODE, http_code); - else - curl_easy_setopt(curl_handle, CURLOPT_FAILONERROR, 1); if(ret != CURLE_OK) { ctx->set_error(ctx, 502, "curl failed to request url %s : %s", req->url, error_msg); diff -Nru mapcache-1.4.0/lib/image.c mapcache-1.6.1/lib/image.c --- mapcache-1.4.0/lib/image.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/image.c 2017-09-29 21:31:13.000000000 +0000 @@ -56,7 +56,7 @@ return img; } -int mapcache_image_has_alpha(mapcache_image *img) +int mapcache_image_has_alpha(mapcache_image *img, unsigned int cutoff) { size_t i,j; if(img->has_alpha == MC_ALPHA_UNKNOWN) { @@ -64,7 +64,7 @@ for(i=0; ih; i++) { ptr = rptr; for(j=0; jw; j++) { - if(ptr[3]<(unsigned char)255) { + if(ptr[3]<(unsigned char)cutoff) { img->has_alpha = MC_ALPHA_YES; return 1; } @@ -85,7 +85,11 @@ void mapcache_image_merge(mapcache_context *ctx, mapcache_image *base, mapcache_image *overlay) { int starti,startj; -#ifndef USE_PIXMAN +#ifdef USE_PIXMAN + pixman_image_t *si; + pixman_image_t *bi; + pixman_transform_t transform; +#else int i,j; unsigned char *browptr, *orowptr, *bptr, *optr; #endif @@ -94,20 +98,22 @@ ctx->set_error(ctx, 500, "attempting to merge an larger image onto another"); return; } + starti = (base->h - overlay->h)/2; startj = (base->w - overlay->w)/2; #ifdef USE_PIXMAN - pixman_image_t *si = pixman_image_create_bits(PIXMAN_a8r8g8b8,overlay->w,overlay->h, + si = pixman_image_create_bits(PIXMAN_a8r8g8b8,overlay->w,overlay->h, (uint32_t*)overlay->data,overlay->stride); - pixman_image_t *bi = pixman_image_create_bits(PIXMAN_a8r8g8b8,base->w,base->h, + bi = pixman_image_create_bits(PIXMAN_a8r8g8b8,base->w,base->h, (uint32_t*)base->data,base->stride); - pixman_transform_t transform; - pixman_transform_init_translate(&transform, - pixman_int_to_fixed(-startj), - pixman_int_to_fixed(-starti)); pixman_image_set_filter(si,PIXMAN_FILTER_NEAREST, NULL, 0); - pixman_image_set_transform (si, &transform); - pixman_image_composite (PIXMAN_OP_OVER, si, si, bi, + if(starti || startj) { + pixman_transform_init_translate(&transform, + pixman_int_to_fixed(-startj), + pixman_int_to_fixed(-starti)); + pixman_image_set_transform (si, &transform); + } + pixman_image_composite (PIXMAN_OP_OVER, si, NULL, bi, 0, 0, 0, 0, 0, 0, base->w,base->h); pixman_image_unref(si); pixman_image_unref(bi); @@ -289,6 +295,12 @@ ctx->set_error(ctx, 500, "failed to load image data from metatile"); return; } + if(metatile->w != mt->map.width || + metatile->h != mt->map.height) { + ctx->set_error(ctx, 500, "image size does not correspond to metatile size"); + return; + } + for(i=0; imetasize_x; i++) { for(j=0; jmetasize_y; j++) { tileimg = mapcache_image_create(ctx); @@ -312,6 +324,9 @@ sx = mt->map.tileset->metabuffer + i * tileimg->w; sy = mt->map.height - (mt->map.tileset->metabuffer + (j+1) * tileimg->h); break; + default: + ctx->set_error(ctx,500,"BUG: unknown grid origin"); + return; } tileimg->data = &(metatile->data[sy*metatile->stride + 4 * sx]); if(mt->map.tileset->watermark) { diff -Nru mapcache-1.4.0/lib/imageio_jpeg.c mapcache-1.6.1/lib/imageio_jpeg.c --- mapcache-1.4.0/lib/imageio_jpeg.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/imageio_jpeg.c 2017-09-29 21:31:13.000000000 +0000 @@ -188,6 +188,18 @@ default: jpeg_set_colorspace(&cinfo, JCS_YCbCr); } + switch(((mapcache_image_format_jpeg*)format)->optimize) { + case MAPCACHE_OPTIMIZE_NO: + cinfo.optimize_coding = FALSE; + break; + case MAPCACHE_OPTIMIZE_ARITHMETIC: + cinfo.optimize_coding = FALSE; + cinfo.arith_code = TRUE; + break; + case MAPCACHE_OPTIMIZE_YES: + default: + cinfo.optimize_coding = TRUE; + } jpeg_start_compress(&cinfo, TRUE); rowdata = (JSAMPLE*)malloc(img->w*cinfo.input_components*sizeof(JSAMPLE)); @@ -317,7 +329,7 @@ } mapcache_image_format* mapcache_imageio_create_jpeg_format(apr_pool_t *pool, char *name, int quality, - mapcache_photometric photometric) + mapcache_photometric photometric, mapcache_optimization optimize) { mapcache_image_format_jpeg *format = apr_pcalloc(pool, sizeof(mapcache_image_format_jpeg)); format->format.name = name; @@ -327,6 +339,7 @@ format->format.create_empty_image = _mapcache_imageio_jpg_create_empty; format->format.write = _mapcache_imageio_jpeg_encode; format->quality = quality; + format->optimize = optimize; format->photometric = photometric; format->format.type = GC_JPEG; return (mapcache_image_format*)format; diff -Nru mapcache-1.4.0/lib/imageio_mixed.c mapcache-1.6.1/lib/imageio_mixed.c --- mapcache-1.4.0/lib/imageio_mixed.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/imageio_mixed.c 2017-09-29 21:31:13.000000000 +0000 @@ -46,7 +46,7 @@ mapcache_image *image, mapcache_image_format *format) { mapcache_image_format_mixed *f = (mapcache_image_format_mixed*)format; - if(mapcache_image_has_alpha(image)) { + if(mapcache_image_has_alpha(image,f->alpha_cutoff)) { return f->transparent->write(ctx,image,f->transparent); } else { return f->opaque->write(ctx,image,f->opaque); @@ -54,12 +54,14 @@ } mapcache_image_format* mapcache_imageio_create_mixed_format(apr_pool_t *pool, - char *name, mapcache_image_format *transparent, mapcache_image_format *opaque) + char *name, mapcache_image_format *transparent, mapcache_image_format *opaque, + unsigned int alpha_cutoff) { mapcache_image_format_mixed *format = apr_pcalloc(pool, sizeof(mapcache_image_format_mixed)); format->format.name = name; format->transparent = transparent; format->opaque = opaque; + format->alpha_cutoff = alpha_cutoff; format->format.extension = apr_pstrdup(pool,"xxx"); format->format.mime_type = NULL; format->format.write = _mapcache_imageio_mixed_encode; diff -Nru mapcache-1.4.0/lib/imageio_png.c mapcache-1.6.1/lib/imageio_png.c --- mapcache-1.4.0/lib/imageio_png.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/imageio_png.c 2017-09-29 21:31:13.000000000 +0000 @@ -324,14 +324,10 @@ // do nothing } -#ifndef _WIN32 -static inline int premultiply (int color,int alpha) -#else -static __inline int premultiply (int color,int alpha) -#endif -{ - int temp = (alpha * color) + 0x80; - return ((temp + (temp >> 8)) >> 8); +#define PREMULTIPLY(out,color,alpha)\ +{\ + int temp = ((alpha) * (color)) + 0x80;\ + out =((temp + (temp >> 8)) >> 8);\ } @@ -425,9 +421,9 @@ pixptr[1] = 0; pixptr[2] = 0; } else { - pixptr[0] = premultiply(pixel[2],alpha); - pixptr[1] = premultiply(pixel[1],alpha); - pixptr[2] = premultiply(pixel[0],alpha); + PREMULTIPLY(pixptr[0],pixel[2],alpha); + PREMULTIPLY(pixptr[1],pixel[1],alpha); + PREMULTIPLY(pixptr[2],pixel[0],alpha); } pixptr += 4; } @@ -540,7 +536,7 @@ png_set_write_fn(png_ptr, buffer, _mapcache_imageio_png_write_func, _mapcache_imageio_png_flush_func); - if(mapcache_image_has_alpha(img)) + if(mapcache_image_has_alpha(img,255)) color_type = PNG_COLOR_TYPE_RGB_ALPHA; else color_type = PNG_COLOR_TYPE_RGB; diff -Nru mapcache-1.4.0/lib/lock.c mapcache-1.6.1/lib/lock.c --- mapcache-1.4.0/lib/lock.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/lock.c 2017-09-29 21:31:13.000000000 +0000 @@ -31,7 +31,27 @@ #include #include #include +#ifndef _WIN32 #include +#endif + +typedef struct { + mapcache_locker locker; + + /** + * directory where lock files will be placed. + * Must be readable and writable by the apache user. + * Must be placed on a network mounted shared directory if multiple mapcache instances + * need to be synchronized + */ + const char *dir; +} mapcache_locker_disk; + +typedef struct { + mapcache_locker locker; + apr_array_header_t *lockers; +} mapcache_locker_fallback; + #define MAPCACHE_LOCKFILE_PREFIX "_gc_lock" @@ -60,16 +80,16 @@ else { apr_time_t start_wait = apr_time_now(); rv = MAPCACHE_LOCK_LOCKED; - + while(rv != MAPCACHE_LOCK_NOENT) { unsigned int waited = apr_time_as_msec(apr_time_now()-start_wait); if(waited > locker->timeout*1000) { - mapcache_unlock_resource(ctx,locker,resource, *lock); + mapcache_unlock_resource(ctx,locker,*lock); ctx->log(ctx,MAPCACHE_ERROR,"deleting a possibly stale lock after waiting on it for %g seconds",waited/1000.0); return MAPCACHE_FALSE; } apr_sleep(locker->retry_interval * 1000000); - rv = locker->ping_lock(ctx,locker,resource, *lock); + rv = locker->ping_lock(ctx,locker, *lock); } return MAPCACHE_FALSE; } @@ -81,12 +101,13 @@ mapcache_locker_disk *ldisk; apr_file_t *lockfile; apr_status_t rv; - + assert(self->type == MAPCACHE_LOCKER_DISK); ldisk = (mapcache_locker_disk*)self; - *lock = NULL; /*unused*/ lockname = lock_filename_for_resource(ctx,ldisk,resource); + *lock = lockname; + /* create the lockfile */ rv = apr_file_open(&lockfile,lockname,APR_WRITE|APR_CREATE|APR_EXCL|APR_XTHREAD,APR_OS_DEFAULT,ctx->pool); @@ -112,12 +133,11 @@ } } -mapcache_lock_result mapcache_locker_disk_ping_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { +mapcache_lock_result mapcache_locker_disk_ping_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { apr_finfo_t info; apr_status_t rv; char *lockname; - mapcache_locker_disk *ldisk = (mapcache_locker_disk*)self; - lockname = lock_filename_for_resource(ctx,ldisk,resource); + lockname = (char*)lock; rv = apr_stat(&info,lockname,0,ctx->pool); if(APR_STATUS_IS_ENOENT(rv)) { return MAPCACHE_LOCK_NOENT; @@ -126,15 +146,14 @@ } } -void mapcache_locker_disk_release_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) +void mapcache_locker_disk_release_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { - mapcache_locker_disk *ld = (mapcache_locker_disk*)self; - char *lockname = lock_filename_for_resource(ctx,ld,resource); + char *lockname = (char*)lock; apr_file_remove(lockname,ctx->pool); } -void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, char *resource, void *lock) { - locker->release_lock(ctx, locker, resource, lock); +void mapcache_unlock_resource(mapcache_context *ctx, mapcache_locker *locker, void *lock) { + locker->release_lock(ctx, locker, lock); } void mapcache_locker_disk_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { @@ -163,14 +182,14 @@ void *lock; /*the opaque lock returned by the locker*/ }; -void mapcache_locker_fallback_release_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { +void mapcache_locker_fallback_release_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { struct mapcache_locker_fallback_lock *flock = lock; - flock->locker->release_lock(ctx,flock->locker,resource,flock->lock); + flock->locker->release_lock(ctx,flock->locker,flock->lock); } -mapcache_lock_result mapcache_locker_fallback_ping_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { +mapcache_lock_result mapcache_locker_fallback_ping_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { struct mapcache_locker_fallback_lock *flock = lock; - return flock->locker->ping_lock(ctx,flock->locker,resource,flock->lock); + return flock->locker->ping_lock(ctx,flock->locker,flock->lock); } mapcache_lock_result mapcache_locker_fallback_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { @@ -188,6 +207,11 @@ fallback_lock->locker = child_locker; ctx->push_errors(ctx,error); return lock_result; + } else { + /*clear the current error if we still have a fallback lock to try */ + if(ilockers->nelts-1) { + ctx->clear_errors(ctx); + } } ctx->push_errors(ctx,error); } @@ -208,6 +232,20 @@ } #ifdef USE_MEMCACHE + +#include + +typedef struct { + char *host; + int port; +} mapcache_locker_memcache_server; + +typedef struct { + mapcache_locker locker; + int nservers; + mapcache_locker_memcache_server *servers; +} mapcache_locker_memcache; + void mapcache_locker_memcache_parse_xml(mapcache_context *ctx, mapcache_locker *self, ezxml_t doc) { mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; ezxml_t node,server_node; @@ -245,7 +283,7 @@ char *saferes = apr_pstrdup(ctx->pool,resource); char *safeptr = saferes; while(*safeptr) { - if(*safeptr==' ' || *safeptr == '/' || *safeptr == '~' || *safeptr == '.' || + if(*safeptr==' ' || *safeptr == '/' || *safeptr == '~' || *safeptr == '.' || *safeptr == '\r' || *safeptr == '\n' || *safeptr == '\t' || *safeptr == '\f' || *safeptr == '\e' || *safeptr == '\a' || *safeptr == '\b') { *safeptr = '#'; } @@ -281,34 +319,38 @@ return memcache; } -mapcache_lock_result mapcache_locker_memcache_ping_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { +typedef struct { + apr_memcache_t *memcache; + char *lockname; +} mapcache_lock_memcache; + +mapcache_lock_result mapcache_locker_memcache_ping_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { apr_status_t rv; char *one; size_t ione; - mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; - char *key = memcache_key_for_resource(ctx, lm, resource); - apr_memcache_t *memcache = (apr_memcache_t*)lock; - if(!memcache) + mapcache_lock_memcache *mlock = (mapcache_lock_memcache*)lock; + if(!mlock || !mlock->lockname || !mlock->memcache) return MAPCACHE_LOCK_NOENT; - rv = apr_memcache_getp(memcache,ctx->pool,key,&one,&ione,NULL); + rv = apr_memcache_getp(mlock->memcache,ctx->pool,mlock->lockname,&one,&ione,NULL); if(rv == APR_SUCCESS) return MAPCACHE_LOCK_LOCKED; else return MAPCACHE_LOCK_NOENT; } - + mapcache_lock_result mapcache_locker_memcache_aquire_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void **lock) { apr_status_t rv; mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; char errmsg[120]; - char *key = memcache_key_for_resource(ctx, lm, resource); - apr_memcache_t *memcache = create_memcache(ctx,lm); + mapcache_lock_memcache *mlock = apr_pcalloc(ctx->pool, sizeof(mapcache_lock_memcache)); + mlock->lockname = memcache_key_for_resource(ctx, lm, resource); + mlock->memcache = create_memcache(ctx,lm); if(GC_HAS_ERROR(ctx)) { return MAPCACHE_LOCK_NOENT; } - *lock = memcache; - rv = apr_memcache_add(memcache,key,"1",1,self->timeout,0); + *lock = mlock; + rv = apr_memcache_add(mlock->memcache,mlock->lockname,"1",1,self->timeout,0); if( rv == APR_SUCCESS) { return MAPCACHE_LOCK_AQUIRED; } else if ( rv == APR_EEXIST ) { @@ -319,20 +361,18 @@ } } -void mapcache_locker_memcache_release_lock(mapcache_context *ctx, mapcache_locker *self, char *resource, void *lock) { +void mapcache_locker_memcache_release_lock(mapcache_context *ctx, mapcache_locker *self, void *lock) { apr_status_t rv; - mapcache_locker_memcache *lm = (mapcache_locker_memcache*)self; + mapcache_lock_memcache *mlock = (mapcache_lock_memcache*)lock; char errmsg[120]; - char *key = memcache_key_for_resource(ctx, lm, resource); - apr_memcache_t *memcache = (apr_memcache_t*)lock; - if(!memcache) { + if(!mlock || !mlock->memcache || !mlock->lockname) { /*error*/ return; } - - rv = apr_memcache_delete(memcache,key,0); + + rv = apr_memcache_delete(mlock->memcache,mlock->lockname,0); if(rv != APR_SUCCESS && rv!= APR_NOTFOUND) { - ctx->set_error(ctx,500,"memcache: failed to delete key %s: %s", key, apr_strerror(rv,errmsg,120)); + ctx->set_error(ctx,500,"memcache: failed to delete key %s: %s", mlock->lockname, apr_strerror(rv,errmsg,120)); } } @@ -363,6 +403,36 @@ return l; } +void mapcache_config_parse_locker_old(mapcache_context *ctx, ezxml_t doc, mapcache_cfg *config) { + /* backwards compatibility */ + int micro_retry; + ezxml_t node; + mapcache_locker_disk *ldisk; + config->locker = mapcache_locker_disk_create(ctx); + ldisk = (mapcache_locker_disk*)config->locker; + if((node = ezxml_child(doc,"lock_dir")) != NULL) { + ldisk->dir = apr_pstrdup(ctx->pool, node->txt); + } else { + ldisk->dir = apr_pstrdup(ctx->pool,"/tmp"); + } + + if((node = ezxml_child(doc,"lock_retry")) != NULL) { + char *endptr; + micro_retry = strtol(node->txt,&endptr,10); + if(*endptr != 0 || micro_retry <= 0) { + ctx->set_error(ctx, 400, "failed to parse lock_retry microseconds \"%s\". Expecting a positive integer", + node->txt); + return; + } + } else { + /* default retry interval is 1/100th of a second, i.e. 10000 microseconds */ + micro_retry = 10000; + } + config->locker->retry_interval = micro_retry / 1000000.0; + config->locker->timeout=120; +} + + void mapcache_config_parse_locker(mapcache_context *ctx, ezxml_t node, mapcache_locker **locker) { ezxml_t cur_node; const char *ltype = ezxml_attr(node, "type"); @@ -383,7 +453,7 @@ return; } (*locker)->parse_xml(ctx, *locker, node); - + if((cur_node = ezxml_child(node,"retry")) != NULL) { char *endptr; (*locker)->retry_interval = strtod(cur_node->txt,&endptr); diff -Nru mapcache-1.4.0/lib/service_demo.c mapcache-1.6.1/lib/service_demo.c --- mapcache-1.4.0/lib/service_demo.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_demo.c 2017-09-29 21:31:13.000000000 +0000 @@ -29,12 +29,11 @@ #include #include "mapcache.h" +#include "mapcache_services.h" #include #include #include -/** \addtogroup services */ -/** @{ */ static char *demo_head = "\n" @@ -282,7 +281,7 @@ " return null;\n" " }\n" "\n" - " // repeat accross x-axis\n" + " // repeat across x-axis\n" " if (x < 0 || x >= tileRange) {\n" " x = (x % tileRange + tileRange) % tileRange;\n" " }\n" @@ -507,7 +506,7 @@ resolutions = apr_psprintf(ctx->pool,"%s,%.20f",resolutions,grid->levels[i]->resolution); } - if(!tileset->timedimension) { + if(!tileset->dimensions) { ol_layer_name = apr_psprintf(ctx->pool, "%s_%s", tileset->name, grid->name); /* normalize name to something that is a valid variable name */ for(i=0; ipool,"%s%s",caps,ol_layer); } } else { - int id; - apr_array_header_t *timedimvals = tileset->timedimension->get_all_entries( - ctx,tileset->timedimension,tileset); - for(id=0;idnelts;id++) { - if(id>1) break; - char *idval = APR_ARRAY_IDX(timedimvals,id,char*); - char *dimparam_wms = " %s_wms_layer.mergeNewParams({%s:\"%s\"});\n"; - char *dimparam_singletile = " %s_slayer.mergeNewParams({%s:\"%s\"});\n"; - ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s", tileset->name, grid->name, idval); - /* normalize name to something that is a valid variable name */ - for(i=0; ipool, demo_layer_wms, - ol_layer_name, - tileset->name, - grid->name, - apr_pstrcat(ctx->pool,url_prefix,"?",NULL), - tileset->name, - resolutions, - unit, - grid->extent.minx, - grid->extent.miny, - grid->extent.maxx, - grid->extent.maxy, - grid->srs, - smerc, - ol_layer_name); - caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); - caps = apr_psprintf(ctx->pool,"%s%s",caps, - apr_psprintf(ctx->pool,dimparam_wms,ol_layer_name,tileset->timedimension->key,idval)); - - if(service->getmap_strategy == MAPCACHE_GETMAP_ASSEMBLE) { - ol_layer = apr_psprintf(ctx->pool, demo_layer_singletile, + int dim; + for(dim=0;dimdimensions->nelts;dim++) { + int id; + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,dim,mapcache_dimension*); + apr_array_header_t *dimvals = dimension->get_all_entries(ctx,dimension,tileset,NULL,NULL); + for(id=0;idnelts;id++) { + char *idval; + char *dimparam_wms; + char *dimparam_singletile; + if(id>1) break; + idval = APR_ARRAY_IDX(dimvals,id,char*); + dimparam_wms = " %s_wms_layer.mergeNewParams({%s:\"%s\"});\n"; + dimparam_singletile = " %s_slayer.mergeNewParams({%s:\"%s\"});\n"; + ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s_%s", tileset->name, grid->name, dimension->name, idval); + /* normalize name to something that is a valid variable name */ + for(i=0; ipool, demo_layer_wms, ol_layer_name, tileset->name, grid->name, @@ -600,7 +584,28 @@ ol_layer_name); caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); caps = apr_psprintf(ctx->pool,"%s%s",caps, - apr_psprintf(ctx->pool,dimparam_singletile,ol_layer_name,tileset->timedimension->key,idval)); + apr_psprintf(ctx->pool,dimparam_wms,ol_layer_name,dimension->name,idval)); + + if(service->getmap_strategy == MAPCACHE_GETMAP_ASSEMBLE) { + ol_layer = apr_psprintf(ctx->pool, demo_layer_singletile, + ol_layer_name, + tileset->name, + grid->name, + apr_pstrcat(ctx->pool,url_prefix,"?",NULL), + tileset->name, + resolutions, + unit, + grid->extent.minx, + grid->extent.miny, + grid->extent.maxx, + grid->extent.maxy, + grid->srs, + smerc, + ol_layer_name); + caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); + caps = apr_psprintf(ctx->pool,"%s%s",caps, + apr_psprintf(ctx->pool,dimparam_singletile,ol_layer_name,dimension->name,idval)); + } } } } @@ -660,15 +665,11 @@ tileindex_index = apr_hash_first(ctx->pool,ctx->config->tilesets); while(tileindex_index) { int i,j; - char *extension; mapcache_tileset *tileset; const void *key; apr_ssize_t keylen; apr_hash_this(tileindex_index,&key,&keylen,(void**)&tileset); - extension = "png"; - if (tileset->format && tileset->format->extension) - extension = tileset->format->extension; for(j=0; jgrid_links->nelts; j++) { char *resolutions=""; char *unit="dd"; @@ -841,7 +842,7 @@ resolutions = apr_psprintf(ctx->pool,"%s,%.20f",resolutions,grid->levels[i]->resolution); } - if(!tileset->timedimension) { + if(!tileset->dimensions) { ol_layer_name = apr_psprintf(ctx->pool, "%s_%s", tileset->name, grid->name); /* normalize name to something that is a valid variable name */ for(i=0; ipool,"%s%s",caps,ol_layer); } else { - int id; - apr_array_header_t *timedimvals = tileset->timedimension->get_all_entries( - ctx,tileset->timedimension,tileset); - GC_CHECK_ERROR(ctx); - for(id=0;idnelts;id++) { - if(id>1) break; /* we don't want all the entries in the demo interface */ - char *idval = APR_ARRAY_IDX(timedimvals,id,char*); - char *dimparam = "%s_wmts_layer.mergeNewParams({%s:\"%s\"});\n"; - ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s", tileset->name, grid->name, idval); - /* normalize name to something that is a valid variable name */ - for(i=0; ipool, demo_layer_wmts, - ol_layer_name, - tileset->name, - grid->name, - apr_pstrcat(ctx->pool,url_prefix,"wmts/",NULL), - tileset->name, - grid->name, - mime_type, - resolutions, - grid_link->minz, - unit, - grid->extent.minx, - grid->extent.miny, - grid->extent.maxx, - grid->extent.maxy, - grid->srs, - smerc, - ol_layer_name); - caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); - caps = apr_psprintf(ctx->pool,"%s%s",caps, - apr_psprintf(ctx->pool,dimparam,ol_layer_name,tileset->timedimension->key,idval)); - + int dim; + for(dim=0;dimdimensions->nelts;dim++) { + int id; + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,dim,mapcache_dimension*); + apr_array_header_t *dimvals = dimension->get_all_entries(ctx,dimension,tileset,NULL,NULL); + GC_CHECK_ERROR(ctx); + for(id=0;idnelts;id++) { + char *idval; + char *dimparam; + if(id>1) break; /* we don't want all the entries in the demo interface */ + idval = APR_ARRAY_IDX(dimvals,id,char*); + dimparam = "%s_wmts_layer.mergeNewParams({%s:\"%s\"});\n"; + ol_layer_name = apr_psprintf(ctx->pool, "%s_%s_%s_%s", tileset->name, grid->name, dimension->name, idval); + /* normalize name to something that is a valid variable name */ + for(i=0; ipool, demo_layer_wmts, + ol_layer_name, + tileset->name, + grid->name, + apr_pstrcat(ctx->pool,url_prefix,"wmts/",NULL), + tileset->name, + grid->name, + mime_type, + resolutions, + grid_link->minz, + unit, + grid->extent.minx, + grid->extent.miny, + grid->extent.maxx, + grid->extent.maxy, + grid->srs, + smerc, + ol_layer_name); + caps = apr_psprintf(ctx->pool,"%s%s",caps,ol_layer); + caps = apr_psprintf(ctx->pool,"%s%s",caps, + apr_psprintf(ctx->pool,dimparam,ol_layer_name,dimension->name,idval)); + + } } } } diff -Nru mapcache-1.4.0/lib/service_kml.c mapcache-1.6.1/lib/service_kml.c --- mapcache-1.4.0/lib/service_kml.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_kml.c 2017-09-29 21:31:13.000000000 +0000 @@ -29,8 +29,10 @@ #include "mapcache.h" #include +#include "mapcache_services.h" #include + /** \addtogroup services */ /** @{ */ @@ -72,7 +74,7 @@ t->x = i; t->y = j; t->z = 0; - mapcache_grid_get_extent(ctx, t->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, t->grid_link->grid, t->x, t->y, t->z, &bb); caps = apr_psprintf(ctx->pool, "%s" @@ -102,7 +104,7 @@ } else { mapcache_extent bbox; - mapcache_grid_get_extent(ctx, request->tile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, request->tile->grid_link->grid, request->tile->x, request->tile->y, request->tile->z, &bbox); @@ -144,7 +146,7 @@ t->x = (request->tile->x << 1) + i; t->y = (request->tile->y << 1) + j; t->z = request->tile->z + 1; - mapcache_grid_get_extent(ctx, t->grid_link->grid, + mapcache_grid_get_tile_extent(ctx, t->grid_link->grid, t->x, t->y, t->z, &bb); caps = apr_psprintf(ctx->pool, "%s" diff -Nru mapcache-1.4.0/lib/service_mapguide.c mapcache-1.6.1/lib/service_mapguide.c --- mapcache-1.4.0/lib/service_mapguide.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_mapguide.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,12 +30,12 @@ #include "mapcache.h" #include #include +#include "mapcache_services.h" /** \addtogroup services */ /** @{ */ - void _create_capabilities_mg(mapcache_context *ctx, mapcache_request_get_capabilities *req, char *url, char *path_info, mapcache_cfg *cfg) { ctx->set_error(ctx, 501, "mapguide service does not support capapbilities"); diff -Nru mapcache-1.4.0/lib/services.c mapcache-1.6.1/lib/services.c --- mapcache-1.4.0/lib/services.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/services.c 2017-09-29 21:31:13.000000000 +0000 @@ -70,6 +70,15 @@ ctx->set_error(ctx,404,"unknown service %s",pathinfo); } +int mapcache_config_services_enabled(mapcache_context *ctx, mapcache_cfg *config) { + int count=0,i; + for(i=0; iservices[i]) + count++; + } + return count; +} + /** @} */ diff -Nru mapcache-1.4.0/lib/service_tms.c mapcache-1.6.1/lib/service_tms.c --- mapcache-1.4.0/lib/service_tms.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_tms.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,7 +30,8 @@ #include "mapcache.h" #include #include -#include +#include "mapcache_services.h" + /** \addtogroup services */ /** @{ */ @@ -174,6 +175,133 @@ } +struct requested_tms_layer{ + mapcache_tileset *tileset; + mapcache_grid_link *grid_link; + apr_array_header_t *dimensions; +}; + +/** + * @brief parse a TMS key representing a layer + * @param ctx + * @param key the string representing the requested layer. can be of the form + * layer + * layer@gridname + * layer[dim1=value1] + * layer[dim1=val1][dim2=val2] + * layer[dim=val]@gridname + * @return + */ +static struct requested_tms_layer* _mapcache_service_tms_parse_layer_key(mapcache_context *ctx, char *key) { + char *grid_name = NULL; + struct requested_tms_layer *rtl = apr_pcalloc(ctx->pool, sizeof(struct requested_tms_layer)); + char *at_ptr = strchr(key,'@'); + char *dim_ptr = strchr(key,'['); + if(!at_ptr && !dim_ptr) { + rtl->tileset = mapcache_configuration_get_tileset(ctx->config,key); + if(!rtl->tileset) { + ctx->set_error(ctx,400,"received TMS with invalid layer name"); + return NULL; + } + return rtl; + } else { + key = apr_pstrdup(ctx->pool,key); + if(at_ptr) { + at_ptr = strchr(key,'@'); + *at_ptr = 0; + } + if(dim_ptr) { + dim_ptr = strchr(key,'['); + *dim_ptr = 0; + } + rtl->tileset = mapcache_configuration_get_tileset(ctx->config,key); + if(!rtl->tileset) { + ctx->set_error(ctx,400,"received TMS with invalid layer name"); + return NULL; + } + /*reapply opening bracket*/ + if(dim_ptr) { + *dim_ptr = '['; + } + } + + /* if here, we have a valid rtl->tileset defined */ + + /* if we have a specific grid requested */ + if(at_ptr) { + int i; + grid_name = at_ptr + 1; + if(!*grid_name) { + ctx->set_error(ctx,400,"received invalid tms layer name. expecting layer_name@grid_name"); + return NULL; + } + for(i=0; itileset->grid_links->nelts; i++) { + mapcache_grid_link *sgrid = APR_ARRAY_IDX(rtl->tileset->grid_links,i,mapcache_grid_link*); + if(!strcmp(sgrid->grid->name,grid_name)) { + rtl->grid_link = sgrid; + break; + } + } + if(!rtl->grid_link) { + ctx->set_error(ctx,400,"received invalid tms layer. grid not configured for requested layer"); + return NULL; + } + } + + /*if we have dimensions requested*/ + if(dim_ptr) { + int i; + if(!rtl->tileset->dimensions || rtl->tileset->dimensions->nelts < 1) { + ctx->set_error(ctx,400,"received invalid tms layer. no dimensions configured for tileset"); + return NULL; + } + for(i=0;itileset->dimensions->nelts;i++) { + mapcache_dimension *dimension = APR_ARRAY_IDX(rtl->tileset->dimensions,i,mapcache_dimension*); + char *individual_dim_ptr; + char *dim_lookup = apr_pstrcat(ctx->pool,"[",dimension->name,"=",NULL); + individual_dim_ptr = strstr(dim_ptr,dim_lookup); + if(individual_dim_ptr) { + mapcache_requested_dimension rdim; + char *dim_value; + char *dim_value_end; + if(!rtl->dimensions) { + rtl->dimensions = apr_array_make(ctx->pool,1,sizeof(mapcache_requested_dimension)); + } + dim_value = individual_dim_ptr + strlen(dim_lookup); + if(!*dim_value || *dim_value == ']') { + ctx->set_error(ctx,400,"received invalid tms layer. failed (1) to parse dimension value"); + return NULL; + } + dim_value_end = strchr(dim_value,']'); + if(!dim_value_end) { + ctx->set_error(ctx,400,"received invalid tms layer. failed (2) to parse dimension value"); + return NULL; + } + *dim_value_end = 0; + rdim.dimension = dimension; + rdim.requested_value = apr_pstrdup(ctx->pool,dim_value); + *dim_value_end = ']'; + APR_ARRAY_PUSH(rtl->dimensions,mapcache_requested_dimension) = rdim; + } + } + /* sanity check: make sure we have as many requested dimensions as '[' characters in key */ + { + int i, count; + if(!rtl->dimensions) { + ctx->set_error(ctx,400,"received invalid tms layer. failed (3) to parse dimension values"); + return NULL; + } + for (i=0, count=0; dim_ptr[i]; i++) + count += (dim_ptr[i] == '['); + if(count != rtl->dimensions->nelts) { + ctx->set_error(ctx,400,"received invalid tms layer. failed (4) to parse dimension values"); + return NULL; + } + } + } + return rtl; +} + /** * \brief parse a TMS request * \private \memberof mapcache_service_tms @@ -185,8 +313,6 @@ int index = 0; char *last, *key, *endptr; char *sTileset = NULL; - mapcache_tileset *tileset = NULL; - mapcache_grid_link *grid_link = NULL; char *pathinfo = NULL; int x=-1,y=-1,z=-1; @@ -238,93 +364,73 @@ } } if(index == 5) { - char *gridname; + char *iter,*gridname=NULL; mapcache_request_get_tile *req = (mapcache_request_get_tile*)apr_pcalloc(ctx->pool,sizeof(mapcache_request_get_tile)); ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; - gridname = sTileset; /*hijack the char* pointer while counting the number of commas */ - while(*gridname) { - if(*gridname == ';') req->ntiles++; - gridname++; + iter = sTileset; + while(*iter) { + if(*iter == ';') req->ntiles++; + iter++; } req->tiles = (mapcache_tile**)apr_pcalloc(ctx->pool,(req->ntiles+1) * sizeof(mapcache_tile*)); - /* reset the hijacked variables back to default value */ - gridname = NULL; req->ntiles = 0; for (key = apr_strtok(sTileset, ";", &last); key != NULL; key = apr_strtok(NULL, ";", &last)) { - tileset = mapcache_configuration_get_tileset(config,key); - if(!tileset) { - /*tileset not found directly, test if it was given as "name@grid" notation*/ - char *tname = apr_pstrdup(ctx->pool,key); - char *gname = tname; - int i; - while(*gname) { - if(*gname == '@') { - *gname = '\0'; - gname++; - break; - } - gname++; - } - if(!gname) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", key); - return; - } - tileset = mapcache_configuration_get_tileset(config,tname); - if(!tileset) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", tname); - return; - } - for(i=0; igrid_links->nelts; i++) { - mapcache_grid_link *sgrid = APR_ARRAY_IDX(tileset->grid_links,i,mapcache_grid_link*); - if(!strcmp(sgrid->grid->name,gname)) { - grid_link = sgrid; - break; - } - } - if(!grid_link) { - ctx->set_error(ctx,404, "received tms request with invalid grid %s", gname); - return; - } - - } else { - grid_link = APR_ARRAY_IDX(tileset->grid_links,0,mapcache_grid_link*); + struct requested_tms_layer *rtl = _mapcache_service_tms_parse_layer_key(ctx,key); + GC_CHECK_ERROR(ctx); + if(!rtl->grid_link) { + rtl->grid_link = APR_ARRAY_IDX(rtl->tileset->grid_links,0,mapcache_grid_link*); } + + /*make sure all our requested layers have the same grid*/ if(!gridname) { - gridname = grid_link->grid->name; + gridname = rtl->grid_link->grid->name; } else { - if(strcmp(gridname,grid_link->grid->name)) { + if(strcmp(gridname,rtl->grid_link->grid->name)) { ctx->set_error(ctx,400,"received tms request with conflicting grids %s and %s", - gridname,grid_link->grid->name); + gridname,rtl->grid_link->grid->name); return; } } if(((mapcache_service_tms*)this)->reverse_y) { - y = grid_link->grid->levels[z]->maxy - y - 1; + y = rtl->grid_link->grid->levels[z]->maxy - y - 1; } - req->tiles[req->ntiles] = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); - switch(grid_link->grid->origin) { + req->tiles[req->ntiles] = mapcache_tileset_tile_create(ctx->pool, rtl->tileset, rtl->grid_link); + switch(rtl->grid_link->grid->origin) { case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: req->tiles[req->ntiles]->x = x; req->tiles[req->ntiles]->y = y; break; case MAPCACHE_GRID_ORIGIN_TOP_LEFT: req->tiles[req->ntiles]->x = x; - req->tiles[req->ntiles]->y = grid_link->grid->levels[z]->maxy - y - 1; + req->tiles[req->ntiles]->y = rtl->grid_link->grid->levels[z]->maxy - y - 1; break; case MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT: - req->tiles[req->ntiles]->x = grid_link->grid->levels[z]->maxx - x - 1; + req->tiles[req->ntiles]->x = rtl->grid_link->grid->levels[z]->maxx - x - 1; req->tiles[req->ntiles]->y = y; break; case MAPCACHE_GRID_ORIGIN_TOP_RIGHT: - req->tiles[req->ntiles]->x = grid_link->grid->levels[z]->maxx - x - 1; - req->tiles[req->ntiles]->y = grid_link->grid->levels[z]->maxy - y - 1; + req->tiles[req->ntiles]->x = rtl->grid_link->grid->levels[z]->maxx - x - 1; + req->tiles[req->ntiles]->y = rtl->grid_link->grid->levels[z]->maxy - y - 1; break; } req->tiles[req->ntiles]->z = z; mapcache_tileset_tile_validate(ctx,req->tiles[req->ntiles]); + if(rtl->dimensions) { + int i; + for(i=0;idimensions->nelts;i++) { + mapcache_requested_dimension rdim = APR_ARRAY_IDX(rtl->dimensions,i,mapcache_requested_dimension); + int j; + for(j=0;jtiles[req->ntiles]->dimensions->nelts;j++) { + mapcache_requested_dimension *tile_dim = APR_ARRAY_IDX(req->tiles[req->ntiles]->dimensions,j,mapcache_requested_dimension*); + if(!strcasecmp(tile_dim->dimension->name,rdim.dimension->name)) { + tile_dim->requested_value = rdim.requested_value; + } + } + } + } req->ntiles++; GC_CHECK_ERROR(ctx); } @@ -335,45 +441,19 @@ ctx->pool,sizeof(mapcache_request_get_capabilities_tms)); req->request.request.type = MAPCACHE_REQUEST_GET_CAPABILITIES; if(index == 2) { - tileset = mapcache_configuration_get_tileset(config,sTileset); - if(!tileset) { - /*tileset not found directly, test if it was given as "name@grid" notation*/ - char *tname = apr_pstrdup(ctx->pool,sTileset); - char *gname = tname; - int i; - while(*gname) { - if(*gname == '@') { - *gname = '\0'; - gname++; - break; - } - gname++; - } - if(!gname) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", key); - return; - } - tileset = mapcache_configuration_get_tileset(config,tname); - if(!tileset) { - ctx->set_error(ctx,404, "received tms request with invalid layer %s", tname); - return; - } - for(i=0; igrid_links->nelts; i++) { - mapcache_grid_link *sgrid = APR_ARRAY_IDX(tileset->grid_links,i,mapcache_grid_link*); - if(!strcmp(sgrid->grid->name,gname)) { - grid_link = sgrid; - break; - } - } - if(!grid_link) { - ctx->set_error(ctx,404, "received tms request with invalid grid %s", gname); - return; - } - } else { - grid_link = APR_ARRAY_IDX(tileset->grid_links,0,mapcache_grid_link*); + struct requested_tms_layer *rtl; + if(strchr(sTileset,';')) { + ctx->set_error(ctx,400,"tms caps: invalid tileset name"); + return; + } + + rtl = _mapcache_service_tms_parse_layer_key(ctx,sTileset); + GC_CHECK_ERROR(ctx); + if(!rtl->grid_link) { + rtl->grid_link = APR_ARRAY_IDX(rtl->tileset->grid_links,0,mapcache_grid_link*); } - req->tileset = tileset; - req->grid_link = grid_link; + req->tileset = rtl->tileset; + req->grid_link = rtl->grid_link; } if(index >= 1) { req->version = apr_pstrdup(ctx->pool,"1.0.0"); diff -Nru mapcache-1.4.0/lib/service_ve.c mapcache-1.6.1/lib/service_ve.c --- mapcache-1.4.0/lib/service_ve.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_ve.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,13 +30,13 @@ #include "mapcache.h" #include #include +#include "mapcache_services.h" /** \addtogroup services */ /** @{ */ - void _create_capabilities_ve(mapcache_context *ctx, mapcache_request_get_capabilities *req, char *url, char *path_info, mapcache_cfg *cfg) { ctx->set_error(ctx, 501, "ve service does not support capapbilities"); @@ -98,32 +98,12 @@ quadkey = apr_table_get(params, "tile"); tile = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); if (quadkey) { - z = strlen(quadkey); + mapcache_util_quadkey_decode(ctx, quadkey, &x, &y, &z); + GC_CHECK_ERROR(ctx); if (z < 1 || z >= grid_link->grid->nlevels) { ctx->set_error(ctx, 404, "received ve request with invalid z level %d\n", z); return; } - x = y = 0; - for (i = z; i; i--) { - int mask = 1 << (i - 1); - switch (quadkey[z - i]) { - case '0': - break; - case '1': - x |= mask; - break; - case '2': - y |= mask; - break; - case '3': - x |= mask; - y |= mask; - break; - default: - ctx->set_error(ctx, 404, "Invalid Quadkey sequence"); - return; - } - } } else { ctx->set_error(ctx, 400, "received ve request with no tile quadkey"); return; diff -Nru mapcache-1.4.0/lib/service_wms.c mapcache-1.6.1/lib/service_wms.c --- mapcache-1.4.0/lib/service_wms.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_wms.c 2017-09-29 21:31:13.000000000 +0000 @@ -30,7 +30,14 @@ #include "mapcache.h" #include #include -#include "ezxml.h" +#include "mapcache_services.h" + +static int sort_strings(const void* pa, const void* pb) +{ + char** ppszA = (char**)pa; + char** ppszB = (char**)pb; + return strcmp(*ppszA, *ppszB); +} /** \addtogroup services */ /** @{ */ @@ -158,7 +165,7 @@ "\n" */ - tmpxml = ezxml_add_child(capxml,"Exceptions",0); + tmpxml = ezxml_add_child(capxml,"Exception",0); ezxml_set_txt(ezxml_add_child(tmpxml,"Format",0),"text/plain"); vendorxml = ezxml_add_child(capxml,"VendorSpecificCapabilities",0); @@ -173,16 +180,28 @@ * * TODO: check for duplicates in gris srs */ - grid_index = apr_hash_first(ctx->pool,cfg->grids); - while(grid_index) { - const void *key; - apr_ssize_t keylen; - mapcache_grid *grid = NULL; - apr_hash_this(grid_index,&key,&keylen,(void**)&grid); - ezxml_set_txt(ezxml_add_child(toplayer,"SRS",0),grid->srs); - grid_index = apr_hash_next(grid_index); - } + { + int srs_count = (int)apr_hash_count(cfg->grids); + char** srs_list = (char**)malloc(srs_count * sizeof(char*)); + int srs_iter = 0; + int i; + grid_index = apr_hash_first(ctx->pool,cfg->grids); + while(grid_index) { + const void *key; + apr_ssize_t keylen; + mapcache_grid *grid = NULL; + apr_hash_this(grid_index,&key,&keylen,(void**)&grid); + srs_list[srs_iter++] = grid->srs; + grid_index = apr_hash_next(grid_index); + } + qsort(srs_list, srs_count, sizeof(char*), sort_strings); + for(i = 0; i < srs_count; i ++) + { + ezxml_set_txt(ezxml_add_child(toplayer,"SRS",0),srs_list[i]); + } + free(srs_list); + } tileindex_index = apr_hash_first(ctx->pool,cfg->tilesets); @@ -200,7 +219,7 @@ layerxml = ezxml_add_child(toplayer,"Layer",0); ezxml_set_attr(layerxml, "cascaded", "1"); ezxml_set_attr(layerxml, "queryable", (tileset->source && tileset->source->info_formats)?"1":"0"); - + ezxml_set_txt(ezxml_add_child(layerxml,"Name",0),tileset->name); tsxml = ezxml_add_child(vendorxml, "TileSet",0); @@ -239,7 +258,7 @@ if(dimension->unit) { ezxml_set_attr(dimxml,"units",dimension->unit); } - values = dimension->print_ogc_formatted_values(ctx,dimension); + values = dimension->get_all_ogc_formatted_entries(ctx,dimension,tileset,NULL,NULL); for(value_idx=0;value_idxnelts;value_idx++) { char *idval = APR_ARRAY_IDX(values,value_idx,char*); if(dimval) { @@ -320,6 +339,26 @@ request->request.mime_type = apr_pstrdup(ctx->pool,"text/xml"); } +static char *_lookup_auto_projection(mapcache_context *ctx, const char *srs) { + if(!strcasecmp(srs,"auto:42001")) { + char *srsdup = apr_pstrdup(ctx->pool,srs); + char *lon = strchr(srsdup,','),*lat; + int nLon,nLat; + if(!lon) return srsdup; + lon = strchr(lon+1,','); + if(!lon) return srsdup; + lon++; + lat = strchr(lon,','); + if(!lat) return srsdup; + *lat = 0; + lat++; + nLon = (int)(floor( (atof(lon) + 180.0) / 6.0 ))*6 + 3 - 180; + nLat = (atof(lat)>=0)?45:-45; + return apr_psprintf(ctx->pool,"auto:42001,9001,%d,%d",nLon,nLat); + } + return (char*)srs; +} + /** * \brief parse a WMS request * \private \memberof mapcache_service_wms @@ -478,7 +517,6 @@ char *last, *layers; const char *key; int count=1; - int nallocated = 0; int i,layeridx; int x,y,z; mapcache_request_get_map *map_req = NULL; @@ -519,6 +557,8 @@ main_tileset->name = (char*)key; } + srs = _lookup_auto_projection(ctx,srs); + for(i=0; igrid_links->nelts; i++) { mapcache_grid_link *sgrid = APR_ARRAY_IDX(main_tileset->grid_links,i,mapcache_grid_link*); /* look for a grid with a matching srs */ @@ -584,7 +624,6 @@ *request = (mapcache_request*)map_req; (*request)->type = MAPCACHE_REQUEST_GET_MAP; } - nallocated = count; /* @@ -599,7 +638,7 @@ int i; mapcache_tileset *tileset = main_tileset; mapcache_grid_link *grid_link = main_grid_link; - apr_table_t *dimtable = NULL; + apr_array_header_t *dimtable = NULL; if(layeridx) { /* @@ -671,94 +710,16 @@ if(strcasecmp(dimension->name,"TIME") && strcasecmp(dimension->name,"ELEVATION")) { /* also test for the dimension without the DIM_ prefix if the latter was not found in the KVP params */ dim_name = dimension->name; - value = (char*)apr_table_get(params,dimension->name); + value = (char*)apr_table_get(params,dim_name); } } if(value) { - char *tmpval = apr_pstrdup(ctx->pool,value); - int ok; - if(dimension->skip_validation) { - ok = MAPCACHE_SUCCESS; - } else { - ok = dimension->validate(ctx,dimension,&tmpval); - GC_CHECK_ERROR(ctx); - } - if(ok == MAPCACHE_SUCCESS) - apr_table_setn(dimtable,dimension->name,tmpval); - else { - errcode = 400; - errmsg = apr_psprintf(ctx->pool, "dimension \"%s\" value \"%s\" fails to validate", - dim_name, value); - goto proxies; - } + mapcache_set_requested_dimension(ctx,dimtable,dimension->name,value); + GC_CHECK_ERROR(ctx); } } } - if(tileset->timedimension) { - /* possibly duplicate the created map/tile for each entry of the requested time dimension */ - apr_array_header_t *timedim_selected; - value = apr_table_get(params,tileset->timedimension->key); - if(!value) - value = tileset->timedimension->default_value; - timedim_selected = mapcache_timedimension_get_entries_for_value(ctx, - tileset->timedimension, tileset, grid_link->grid, &extent, value); - GC_CHECK_ERROR(ctx); - if(!timedim_selected || timedim_selected->nelts == 0) { - errcode = 404; - errmsg = apr_psprintf(ctx->pool,"no matching entry for given TIME dimension \"%s\" in tileset \"%s\"", - tileset->timedimension->key, tileset->name); - goto proxies; - } - if(type == MAPCACHE_REQUEST_GET_TILE) { - int i; - /* we need to create more tile/map entries */ - if(timedim_selected->nelts > 1) { - /* apr pools have no realloc */ - nallocated = nallocated + timedim_selected->nelts - 1; - mapcache_tile** tmptiles = - apr_palloc(ctx->pool, nallocated * sizeof(mapcache_tile*)); - for(i=0;intiles;i++) { - tmptiles[i] = tile_req->tiles[i]; - } - tile_req->tiles = tmptiles; - /* end realloc workaround */ - } - for(i=0;inelts;i++) { - if(i) { - tile_req->tiles[tile_req->ntiles] = - mapcache_tileset_tile_clone(ctx->pool,tile_req->tiles[tile_req->ntiles-1]); - tile_req->ntiles++; - } - apr_table_set(tile_req->tiles[tile_req->ntiles-1]->dimensions,tileset->timedimension->key, - APR_ARRAY_IDX(timedim_selected,i,char*)); - } - } else { - int i; - /* we need to create more tile/map entries */ - if(timedim_selected->nelts > 1) { - /* apr pools have no realloc */ - nallocated = nallocated + timedim_selected->nelts - 1; - mapcache_map** tmpmaps = - apr_palloc(ctx->pool, nallocated * sizeof(mapcache_map*)); - for(i=0;inmaps;i++) { - tmpmaps[i] = map_req->maps[i]; - } - map_req->maps = tmpmaps; - /* end realloc workaround */ - } - for(i=0;inelts;i++) { - if(i) { - map_req->maps[map_req->nmaps] = - mapcache_tileset_map_clone(ctx->pool,map_req->maps[map_req->nmaps-1]); - map_req->nmaps++; - } - apr_table_set(map_req->maps[map_req->nmaps-1]->dimensions,tileset->timedimension->key, - APR_ARRAY_IDX(timedim_selected,i,char*)); - } - - } - } } } if(tile_req && tile_req->ntiles == 0) { @@ -856,17 +817,8 @@ mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); const char *value; if((value = (char*)apr_table_get(params,dimension->name)) != NULL) { - char *tmpval = apr_pstrdup(ctx->pool,value); - int ok = dimension->validate(ctx,dimension,&tmpval); + mapcache_map_set_cached_dimension(ctx,&fi->map,dimension->name,value); GC_CHECK_ERROR(ctx); - if(ok == MAPCACHE_SUCCESS) - apr_table_setn(fi->map.dimensions,dimension->name,tmpval); - else { - errcode = 400; - errmsg = apr_psprintf(ctx->pool,"dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); - goto proxies; - } } } } @@ -908,7 +860,7 @@ for(j=0; jmatch_params->nelts; j++) { mapcache_dimension *match_param = APR_ARRAY_IDX(rule->match_params,j,mapcache_dimension*); const char *value = apr_table_get(params,match_param->name); - if(!value || match_param->validate(ctx,match_param,(char**)&value) == MAPCACHE_FAILURE) { + if(!value || match_param->get_entries_for_value(ctx,match_param,value,NULL,NULL,NULL)->nelts == 0) { /* the parameter was not supplied, or did not validate: we don't apply this rule */ got_a_match = 0; break; @@ -965,7 +917,7 @@ rule = apr_pcalloc(ctx->pool, sizeof(mapcache_forwarding_rule)); rule->name = apr_pstrdup(ctx->pool,name); rule->match_params = apr_array_make(ctx->pool,1,sizeof(mapcache_dimension*)); - rule->max_post_len = 10485760; /* 10 megabytes by default */ + rule->max_post_len = 10485760; /* 10 megabytes by default */ node = ezxml_child(rule_node,"append_pathinfo"); if(node && !strcasecmp(node->txt,"true")) { @@ -973,7 +925,7 @@ } else { rule->append_pathinfo = 0; } - + node = ezxml_child(rule_node,"max_post_length"); if(node) { char *endptr; @@ -983,7 +935,7 @@ return; } } - + node = ezxml_child(rule_node,"http"); if(!node) { ctx->set_error(ctx,500,"rule \"%s\" does not contain an block",name); @@ -1005,9 +957,9 @@ if(type && *type) { if(!strcmp(type,"values")) { - dimension = mapcache_dimension_values_create(ctx->pool); + dimension = mapcache_dimension_values_create(ctx,ctx->pool); } else if(!strcmp(type,"regex")) { - dimension = mapcache_dimension_regex_create(ctx->pool); + dimension = mapcache_dimension_regex_create(ctx,ctx->pool); } else { ctx->set_error(ctx,400,"unknown type \"%s\". expecting \"values\" or \"regex\".",type); return; @@ -1016,6 +968,7 @@ ctx->set_error(ctx,400, "mandatory attribute \"type\" not found in "); return; } + GC_CHECK_ERROR(ctx); dimension->name = apr_pstrdup(ctx->pool,name); @@ -1080,9 +1033,7 @@ \"http://schemas.opengis.net/wms/1.1.1/exception_1_1_1.dtd\">\n\ \n\ \n\ -\n\ \n\ %s\ "; @@ -1099,7 +1050,9 @@ } } - *err_body = apr_psprintf(ctx->pool,template,msg,exceptions); + *err_body = apr_psprintf(ctx->pool,template, + mapcache_util_str_xml_escape(ctx->pool, msg, MAPCACHE_UTIL_XML_SECTION_TEXT), + exceptions); apr_table_set(headers, "Content-Type", "application/vnd.ogc.se_xml"); } diff -Nru mapcache-1.4.0/lib/service_wmts.c mapcache-1.6.1/lib/service_wmts.c --- mapcache-1.4.0/lib/service_wmts.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/service_wmts.c 2017-09-29 21:31:13.000000000 +0000 @@ -31,13 +31,12 @@ #include #include #include -#include "ezxml.h" +#include "mapcache_services.h" /** \addtogroup services */ /** @{ */ - static ezxml_t _wmts_capabilities(mapcache_context *ctx, mapcache_cfg *cfg) { char *schemaLocation = "http://www.opengis.net/wmts/1.0 http://schemas.opengis.net/wmts/1.0/wmtsGetCapabilities_response.xsd"; @@ -382,7 +381,7 @@ if(dimension->unit) { ezxml_set_txt(ezxml_add_child(dim,"UOM",0),dimension->unit); } - values = dimension->print_ogc_formatted_values(ctx,dimension); + values = dimension->get_all_ogc_formatted_entries(ctx,dimension,tileset,NULL,NULL); for(value_idx=0;value_idxnelts;value_idx++) { char *idval = APR_ARRAY_IDX(values,value_idx,char*); ezxml_set_txt(ezxml_add_child(dim,"Value",0),idval); @@ -417,13 +416,13 @@ ezxml_set_txt(ezxml_add_child(matrixlimits,"TileMatrix",0), apr_psprintf(ctx->pool,"%s:%d",grid_link->grid->name,j)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MinTileRow",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].minx)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].miny)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MaxTileRow",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxx-1)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxy-1)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MinTileCol",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].miny)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].minx)); ezxml_set_txt(ezxml_add_child(matrixlimits,"MaxTileCol",0), - apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxy-1)); + apr_psprintf(ctx->pool,"%d",grid_link->grid_limits[j].maxx-1)); } } @@ -497,10 +496,18 @@ bbox = ezxml_add_child(tmset,"ows:BoundingBox",0); - ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0),apr_psprintf(ctx->pool,"%f %f", - grid->extent.minx, grid->extent.miny)); - ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0),apr_psprintf(ctx->pool,"%f %f", - grid->extent.maxx, grid->extent.maxy)); + if(mapcache_is_axis_inverted(grid->srs)) { + ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.miny, grid->extent.minx)); + ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.maxy, grid->extent.maxx)); + } else { + ezxml_set_txt(ezxml_add_child(bbox,"ows:LowerCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.minx, grid->extent.miny)); + ezxml_set_txt(ezxml_add_child(bbox,"ows:UpperCorner",0),apr_psprintf(ctx->pool,"%f %f", + grid->extent.maxx, grid->extent.maxy)); + } + ezxml_set_attr(bbox,"crs",mapcache_grid_get_crs(ctx,grid)); ezxml_set_txt(ezxml_add_child(tmset,"ows:SupportedCRS",0),mapcache_grid_get_crs(ctx,grid)); @@ -527,6 +534,7 @@ break; case MAPCACHE_GRID_ORIGIN_BOTTOM_RIGHT: case MAPCACHE_GRID_ORIGIN_TOP_RIGHT: + default: ctx->set_error(ctx,500,"origin not implemented"); return; } @@ -566,8 +574,6 @@ *infoformat = NULL, *fi_i = NULL, *fi_j = NULL; apr_table_t *dimtable = NULL; mapcache_extent *extent = NULL; - char *timedim = NULL; - apr_array_header_t *timedim_selected; /* the individual time entries that corresponded to the input request */ mapcache_tileset *tileset = NULL; int row,col,level,x,y; int kvp = 0; @@ -630,14 +636,6 @@ } } } - if(tileset->timedimension) { - const char* value; - if((value = apr_table_get(params,tileset->timedimension->key)) != NULL) { - timedim = apr_pstrdup(ctx->pool,value); - } else { - timedim = apr_pstrdup(ctx->pool,tileset->timedimension->default_value); - } - } if(!strcasecmp(str,"getfeatureinfo")) { infoformat = apr_table_get(params,"INFOFORMAT"); fi_i = apr_table_get(params,"I"); @@ -702,10 +700,6 @@ continue; } } - if(!timedim && tileset->timedimension) { - timedim = key; - continue; - } if(!matrixset) { matrixset = key; continue; @@ -768,41 +762,9 @@ return; } - /*validate dimensions*/ - if(tileset->dimensions) { - int i; - if(!dimtable) { - ctx->set_error(ctx,404, "received request with no dimensions"); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","dim"); - return; - } - - for(i=0; idimensions->nelts; i++) { - char *tmpval; - int ok; - const char *value; - mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - if(dimension->skip_validation) continue; - value = apr_table_get(dimtable,dimension->name); - if(value) { - tmpval = apr_pstrdup(ctx->pool,value); - ok = dimension->validate(ctx,dimension,&tmpval); - GC_CHECK_ERROR(ctx); - if(ok != MAPCACHE_SUCCESS) { - ctx->set_error(ctx,404,"dimension \"%s\" value \"%s\" fails to validate", - dimension->name, value); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","%s",dimension->name); - return; - } - /* re-set the eventually modified value in the dimension table */ - apr_table_set(dimtable,dimension->name,tmpval); - } - } - } - if(!matrixset) { ctx->set_error(ctx, 404, "received wmts request with no TILEMATRIXSET"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilematrixset"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileMatrixSet"); return; } else { int i; @@ -814,49 +776,49 @@ } if(!grid_link) { ctx->set_error(ctx, 404, "received wmts request with invalid TILEMATRIXSET %s",matrixset); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilematrixset"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileMatrixSet"); return; } } if(!matrix) { ctx->set_error(ctx, 404, "received wmts request with no TILEMATRIX"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilematrix"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileMatrix"); return; } else { char *endptr; level = (int)strtol(matrix,&endptr,10); if(*endptr != 0 || level < grid_link->minz || level >= grid_link->maxz) { ctx->set_error(ctx, 404, "received wmts request with invalid TILEMATRIX %s", matrix); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilematrix"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileMatrix"); return; } } if(!tilerow) { ctx->set_error(ctx, 404, "received wmts request with no TILEROW"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilerow"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileRow"); return; } else { char *endptr; row = (int)strtol(tilerow,&endptr,10); if(*endptr != 0 || row < 0) { ctx->set_error(ctx, 404, "received wmts request with invalid TILEROW %s",tilerow); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilerow"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileRow"); return; } } if(!tilecol) { ctx->set_error(ctx, 404, "received wmts request with no TILECOL"); - if(kvp) ctx->set_exception(ctx,"MissingParameterValue","tilecol"); + if(kvp) ctx->set_exception(ctx,"MissingParameterValue","TileCol"); return; } else { char *endptr; col = (int)strtol(tilecol,&endptr,10); if(endptr == tilecol || col < 0) { ctx->set_error(ctx, 404, "received wmts request with invalid TILECOL %s",tilecol); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","tilecol"); + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileCol"); return; } } @@ -879,17 +841,19 @@ x = grid_link->grid->levels[level]->maxx - col - 1; y = row; break; + default: + ctx->set_error(ctx,500,"BUG: invalid grid origin"); + return; } - if(fi_j || timedim) { + if(fi_j) { //we need the extent of the request, compute it here extent = apr_pcalloc(ctx->pool, sizeof(mapcache_extent)); - mapcache_grid_get_extent(ctx,grid_link->grid,x,y,level,extent); + mapcache_grid_get_tile_extent(ctx,grid_link->grid,x,y,level,extent); } if(!fi_j) { /*we have a getTile request*/ - int i; #ifdef PEDANTIC_WMTS_FORMAT_CHECK if(tileset->format) { @@ -916,57 +880,46 @@ ctx->pool,sizeof(mapcache_request_get_tile)); ((mapcache_request*)req)->type = MAPCACHE_REQUEST_GET_TILE; - if(timedim) { - timedim_selected = mapcache_timedimension_get_entries_for_value(ctx, - tileset->timedimension, tileset, grid_link->grid, extent, timedim); - GC_CHECK_ERROR(ctx); - if(!timedim_selected || timedim_selected->nelts == 0) { - ctx->set_error(ctx, 404, "no matching entry for given TIME dimension"); - if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TIME"); - return; - } - req->ntiles = timedim_selected->nelts; - } else { - req->ntiles = 1; - } + req->ntiles = 1; req->tiles = (mapcache_tile**)apr_pcalloc(ctx->pool,req->ntiles * sizeof(mapcache_tile*)); - for(i=0;intiles;i++) { - req->tiles[i] = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); - if(!req->tiles[i]) { - ctx->set_error(ctx, 500, "failed to allocate tile"); - if(kvp) ctx->set_exception(ctx,"NoApplicableCode",""); - return; - } - - /*populate dimensions*/ - if(tileset->dimensions) { - int d; - for(d=0; ddimensions->nelts; d++) { - mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,d,mapcache_dimension*); - const char *value = apr_table_get(dimtable,dimension->name); - if(value) { - apr_table_set(req->tiles[i]->dimensions,dimension->name,value); - } - } - } - if(tileset->timedimension) { - apr_table_set(req->tiles[i]->dimensions,tileset->timedimension->key, - APR_ARRAY_IDX(timedim_selected,i,char*)); - } + req->tiles[0] = mapcache_tileset_tile_create(ctx->pool, tileset, grid_link); + if(!req->tiles[0]) { + ctx->set_error(ctx, 500, "failed to allocate tile"); + if(kvp) ctx->set_exception(ctx,"NoApplicableCode",""); + return; + } - req->tiles[i]->z = level; - req->tiles[i]->x = x; - req->tiles[i]->y = y; - if(i==0) { - /* no need to validate all the tiles as they all have the same x,y,z */ - mapcache_tileset_tile_validate(ctx,req->tiles[0]); - if(GC_HAS_ERROR(ctx)) { - if(kvp) ctx->set_exception(ctx,"TileOutOfRange",""); - return; + /*populate dimensions*/ + if(tileset->dimensions) { + int d; + for(d=0; ddimensions->nelts; d++) { + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,d,mapcache_dimension*); + const char *value = apr_table_get(dimtable,dimension->name); + if(value) { + mapcache_tile_set_requested_dimension(ctx,req->tiles[0],dimension->name,value); } } } + req->tiles[0]->z = level; + req->tiles[0]->x = x; + req->tiles[0]->y = y; + /* no need to validate all the tiles as they all have the same x,y,z */ + mapcache_tileset_tile_validate_z(ctx,req->tiles[0]); + if(GC_HAS_ERROR(ctx)) { + if(kvp) ctx->set_exception(ctx,"InvalidParameterValue","TileMatrix"); + return; + } + mapcache_tileset_tile_validate_x(ctx,req->tiles[0]); + if(GC_HAS_ERROR(ctx)) { + if(kvp) ctx->set_exception(ctx,"TileOutOfRange","TileCol"); + return; + } + mapcache_tileset_tile_validate_y(ctx,req->tiles[0]); + if(GC_HAS_ERROR(ctx)) { + if(kvp) ctx->set_exception(ctx,"TileOutOfRange","TileRow"); + return; + } *request = (mapcache_request*)req; return; @@ -1023,7 +976,7 @@ mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,d,mapcache_dimension*); const char *value = apr_table_get(dimtable,dimension->name); if(value) { - apr_table_set(fi->map.dimensions,dimension->name,value); + mapcache_map_set_requested_dimension(ctx,&fi->map,dimension->name,value); } } } @@ -1061,7 +1014,9 @@ "",elts[i].key,elts[i].val),NULL); } - *err_body = apr_psprintf(ctx->pool,template,msg,exceptions); + *err_body = apr_psprintf(ctx->pool,template, + mapcache_util_str_xml_escape(ctx->pool, msg, MAPCACHE_UTIL_XML_SECTION_COMMENT), + exceptions); apr_table_set(headers, "Content-Type", "application/xml"); diff -Nru mapcache-1.4.0/lib/source.c mapcache-1.6.1/lib/source.c --- mapcache-1.4.0/lib/source.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/source.c 2017-09-29 21:31:13.000000000 +0000 @@ -28,6 +28,7 @@ *****************************************************************************/ #include "mapcache.h" +#include @@ -36,6 +37,60 @@ mapcache_extent tmp_extent = {-1,-1,-1,-1}; source->data_extent = tmp_extent; source->metadata = apr_table_make(ctx->pool,3); + source->retry_count = 1; + source->retry_delay = 0.1; +} + + +void mapcache_source_render_map(mapcache_context *ctx, mapcache_source *source, mapcache_map *map) { + int i; +#ifdef DEBUG + ctx->log(ctx, MAPCACHE_DEBUG, "calling render_map on source (%s): tileset=%s, grid=%s, extent=(%f,%f,%f,%f)", + source->name, map->tileset->name, map->grid_link->grid->name, + map->extent.minx, map->extent.miny, map->extent.maxx, map->extent.maxy); +#endif + for(i=0;i<=source->retry_count;i++) { + if(i) { /* not our first try */ + ctx->log(ctx, MAPCACHE_INFO, "source (%s) render_map retry %d of %d. previous try returned error: %s", + source->name, i, source->retry_count, ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(source->retry_delay > 0) { + double wait = source->retry_delay; + int j = 0; + for(j=1;j_render_map(ctx, source, map); + if(!GC_HAS_ERROR(ctx)) + break; + } +} + +void mapcache_source_query_info(mapcache_context *ctx, mapcache_source *source, mapcache_feature_info *fi) { + int i; +#ifdef DEBUG + ctx->log(ctx, MAPCACHE_DEBUG, "calling query_info on source (%s): tileset=%s, grid=%s,", + source->name, fi->map.tileset->name, fi->map.grid_link->grid->name); +#endif + for(i=0;i<=source->retry_count;i++) { + if(i) { /* not our first try */ + ctx->log(ctx, MAPCACHE_INFO, "source (%s) render_map retry %d of %d. previous try returned error: %s", + source->name, i, source->retry_count, ctx->get_error_message(ctx)); + ctx->clear_errors(ctx); + if(source->retry_delay > 0) { + double wait = source->retry_delay; + int j = 0; + for(j=1;j_query_info(ctx, source, fi); + if(!GC_HAS_ERROR(ctx)) + break; + } } /* vim: ts=2 sts=2 et sw=2 */ diff -Nru mapcache-1.4.0/lib/source_dummy.c mapcache-1.6.1/lib/source_dummy.c --- mapcache-1.4.0/lib/source_dummy.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/source_dummy.c 2017-09-29 21:31:13.000000000 +0000 @@ -33,11 +33,18 @@ #include #include +typedef struct mapcache_source_dummy mapcache_source_dummy; +struct mapcache_source_dummy { + mapcache_source source; + char *mapfile; + void *mapobj; +}; + /** * \private \memberof mapcache_source_dummy * \sa mapcache_source::render_map() */ -void _mapcache_source_dummy_render_map(mapcache_context *ctx, mapcache_map *map) +void _mapcache_source_dummy_render_map(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) { map->raw_image = mapcache_image_create(ctx); map->raw_image->w = map->width; @@ -48,7 +55,7 @@ apr_pool_cleanup_register(ctx->pool, map->raw_image->data,(void*)free, apr_pool_cleanup_null); } -void _mapcache_source_dummy_query(mapcache_context *ctx, mapcache_feature_info *fi) +void _mapcache_source_dummy_query(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi) { ctx->set_error(ctx,500,"dummy source does not support queries"); } @@ -57,7 +64,7 @@ * \private \memberof mapcache_source_dummy * \sa mapcache_source::configuration_parse() */ -void _mapcache_source_dummy_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source) +void _mapcache_source_dummy_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source, mapcache_cfg *config) { } @@ -79,10 +86,10 @@ } mapcache_source_init(ctx, &(source->source)); source->source.type = MAPCACHE_SOURCE_DUMMY; - source->source.render_map = _mapcache_source_dummy_render_map; + source->source._render_map = _mapcache_source_dummy_render_map; source->source.configuration_check = _mapcache_source_dummy_configuration_check; source->source.configuration_parse_xml = _mapcache_source_dummy_configuration_parse_xml; - source->source.query_info = _mapcache_source_dummy_query; + source->source._query_info = _mapcache_source_dummy_query; return (mapcache_source*)source; } diff -Nru mapcache-1.4.0/lib/source_fallback.c mapcache-1.6.1/lib/source_fallback.c --- mapcache-1.4.0/lib/source_fallback.c 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/lib/source_fallback.c 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,165 @@ +/****************************************************************************** + * $Id$ + * + * Project: MapServer + * Purpose: MapCache tile caching support file: Mapserver Mapfile datasource + * Author: Thomas Bonfort and the MapServer team. + * + ****************************************************************************** + * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies of this Software or works derived from this Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + *****************************************************************************/ + + +#include "mapcache.h" +#include "ezxml.h" +#include +#include + +typedef struct mapcache_source_fallback mapcache_source_fallback; +struct mapcache_source_fallback { + mapcache_source source; + apr_array_header_t *sources; +}; + +/** + * \private \memberof mapcache_source_fallback + * \sa mapcache_source::render_map() + */ +void _mapcache_source_fallback_render_map(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) +{ + mapcache_source_fallback *source = (mapcache_source_fallback*)psource; + mapcache_source *subsource; + int i; + subsource = APR_ARRAY_IDX(source->sources,0,mapcache_source*); + mapcache_source_render_map(ctx, subsource, map); + + if(GC_HAS_ERROR(ctx)) { + int first_error = ctx->get_error(ctx); + char *first_error_message = ctx->get_error_message(ctx); + ctx->log(ctx,MAPCACHE_INFO, + "failed render on primary source \"%s\" on tileset \"%s\". Falling back on secondary sources", + subsource->name,map->tileset->name); + ctx->clear_errors(ctx); + for(i=1; isources->nelts; i++) { + subsource = APR_ARRAY_IDX(source->sources,i,mapcache_source*); + mapcache_source_render_map(ctx, subsource, map); + if(GC_HAS_ERROR(ctx)) { + ctx->log(ctx,MAPCACHE_INFO, + "failed render on fallback source \"%s\" of tileset \"%s\". Continuing with other fallback sources if available", + subsource->name,map->tileset->name); + ctx->clear_errors(ctx); + continue; + } else { + return; + } + } + /* all backends failed, return primary error message */ + ctx->set_error(ctx,first_error,first_error_message); + return; + } +} + +void _mapcache_source_fallback_query(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi) +{ + mapcache_source_fallback *source = (mapcache_source_fallback*)psource; + mapcache_source *subsource; + int i; + subsource = APR_ARRAY_IDX(source->sources,0,mapcache_source*); + mapcache_source_query_info(ctx, subsource, fi); + + if(GC_HAS_ERROR(ctx)) { + int first_error = ctx->get_error(ctx); + char *first_error_message = ctx->get_error_message(ctx); + ctx->log(ctx,MAPCACHE_INFO, + "failed query_info on primary source \"%s\" on tileset \"%s\". Falling back on secondary sources", + subsource->name,fi->map.tileset->name); + ctx->clear_errors(ctx); + for(i=1; isources->nelts; i++) { + subsource = APR_ARRAY_IDX(source->sources,i,mapcache_source*); + mapcache_source_query_info(ctx, subsource, fi); + if(GC_HAS_ERROR(ctx)) { + ctx->log(ctx,MAPCACHE_INFO, + "failed query_info on fallback source \"%s\" of tileset \"%s\". Continuing with other fallback sources if available", + subsource->name,fi->map.tileset->name); + ctx->clear_errors(ctx); + continue; + } else { + return; + } + } + /* all backends failed, return primary error message */ + ctx->set_error(ctx,first_error,first_error_message); + return; + } + ctx->set_error(ctx,500,"fallback source does not support queries"); +} + +/** + * \private \memberof mapcache_source_fallback + * \sa mapcache_source::configuration_parse() + */ +void _mapcache_source_fallback_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *psource, mapcache_cfg *config) +{ + ezxml_t cur_node; + mapcache_source_fallback *source = (mapcache_source_fallback*)psource; + source->sources = apr_array_make(ctx->pool,3,sizeof(mapcache_source*)); + for(cur_node = ezxml_child(node,"source"); cur_node; cur_node = cur_node->next) { + mapcache_source *refsource = mapcache_configuration_get_source(config, cur_node->txt); + if(!refsource) { + ctx->set_error(ctx, 400, "fallback source \"%s\" references source \"%s\"," + " but it is not configured (hint:referenced sources must be declared before this fallback source in the xml file)", psource->name, cur_node->txt); + return; + } + APR_ARRAY_PUSH(source->sources,mapcache_source*) = refsource; + } + if(source->sources->nelts == 0) { + ctx->set_error(ctx,400,"fallback source \"%s\" does not reference any child sources", psource->name); + } +} + +/** + * \private \memberof mapcache_source_fallback + * \sa mapcache_source::configuration_check() + */ +void _mapcache_source_fallback_configuration_check(mapcache_context *ctx, mapcache_cfg *cfg, + mapcache_source *source) +{ +} + +mapcache_source* mapcache_source_fallback_create(mapcache_context *ctx) +{ + mapcache_source_fallback *source = apr_pcalloc(ctx->pool, sizeof(mapcache_source_fallback)); + if(!source) { + ctx->set_error(ctx, 500, "failed to allocate fallback source"); + return NULL; + } + mapcache_source_init(ctx, &(source->source)); + source->source.type = MAPCACHE_SOURCE_FALLBACK; + source->source._render_map = _mapcache_source_fallback_render_map; + source->source.configuration_check = _mapcache_source_fallback_configuration_check; + source->source.configuration_parse_xml = _mapcache_source_fallback_configuration_parse_xml; + source->source._query_info = _mapcache_source_fallback_query; + return (mapcache_source*)source; +} + + +/* vim: ts=2 sts=2 et sw=2 +*/ diff -Nru mapcache-1.4.0/lib/source_gdal.c mapcache-1.6.1/lib/source_gdal.c --- mapcache-1.4.0/lib/source_gdal.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/source_gdal.c 2017-09-29 21:31:13.000000000 +0000 @@ -3,10 +3,11 @@ * * Project: MapServer * Purpose: MapCache tile caching support file: GDAL datasource support (incomplete and disabled) - * Author: Thomas Bonfort and the MapServer team. + * Author: Thomas Bonfort, Even Rouault and the MapServer team. * ****************************************************************************** * Copyright (c) 1996-2011 Regents of the University of Minnesota. + * Copyright (c) 2004, Frank Warmerdam (for GDALAutoCreateWarpedVRT who CreateWarpedVRT is derived from) * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), @@ -37,190 +38,573 @@ #include #include -#include "gdal_alg.h" +#include "gdalwarper.h" #include "cpl_string.h" #include "ogr_srs_api.h" +#include "gdal_vrt.h" -/** - * \private \memberof mapcache_source_gdal - * \sa mapcache_source::render_metatile() - */ -void _mapcache_source_gdal_render_metatile(mapcache_context *ctx, mapcache_metatile *tile) -{ - mapcache_source_gdal *gdal = (mapcache_source_gdal*)tile->tile.tileset->source; - char *srcSRS = "", *dstSRS; - mapcache_buffer *data = mapcache_buffer_create(0,ctx->pool); - GC_CHECK_ERROR(ctx); - GDALDatasetH hDataset; - - GDALAllRegister(); - OGRSpatialReferenceH hSRS; - CPLErrorReset(); +#define MAPCACHE_DEFAULT_RESAMPLE_ALG GRA_Bilinear - hSRS = OSRNewSpatialReference( NULL ); - if( OSRSetFromUserInput( hSRS, tile->tile.grid->srs ) == OGRERR_NONE ) - OSRExportToWkt( hSRS, &dstSRS ); - else { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"failed to parse gdal srs %s",tile->tile.grid->srs); - return; - } +/* Note: this will also work with GDAL >= 2.0 */ +#if GDAL_VERSION_MAJOR < 2 +#define USE_PRE_GDAL2_METHOD +#endif +/*#define USE_PRE_GDAL2_METHOD*/ - OSRDestroySpatialReference( hSRS ); +typedef struct mapcache_source_gdal mapcache_source_gdal; - hDataset = GDALOpen( gdal->datastr, GA_ReadOnly ); - if( hDataset == NULL ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"GDAL failed to open %s",gdal->datastr); +/**\class mapcache_source_gdal + * \brief GDAL mapcache_source + * \implements mapcache_source + */ +struct mapcache_source_gdal { + mapcache_source source; + const char *datastr; /**< the gdal source string*/ + const char *srs_wkt; + GDALResampleAlg eResampleAlg; /**< resampling algorithm */ + const char *srcOvrLevel; /**< strategy to pickup source overview: AUTO, NONE, + AUTO-xxx, xxxx. See -ovr doc in http://www.gdal.org/gdalwarp.html. + Only used for GDAL >= 2.0 (could probably be made to work for USE_PRE_GDAL2_METHOD with more work) */ + int bUseConnectionPool; +}; + +typedef struct { + mapcache_source_gdal *gdal; + const char *gdal_data; + const char *dst_srs; +} gdal_connection_params; + +typedef struct { + GDALDatasetH hSrcDS; + char *dst_srs_wkt; +} gdal_connection; + +void mapcache_source_gdal_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { + gdal_connection_params *p = (gdal_connection_params*)params; + gdal_connection *c = malloc(sizeof(gdal_connection)); + OGRSpatialReferenceH hDstSRS; + + *conn_ = NULL; + /* -------------------------------------------------------------------- */ + /* Open source dataset. */ + /* -------------------------------------------------------------------- */ + c->hSrcDS = GDALOpen( p->gdal_data, GA_ReadOnly ); + + if( c->hSrcDS == NULL ) { + ctx->set_error(ctx, 500, "Cannot open gdal source for %s .\n", p->gdal->source.name ); + free(c); return; } /* -------------------------------------------------------------------- */ - /* Check that there's at least one raster band */ + /* Check that there's 3 or 4 raster bands. */ /* -------------------------------------------------------------------- */ - if ( GDALGetRasterCount(hDataset) == 0 ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"raster %s has no bands",gdal->datastr); + if ( GDALGetRasterCount(c->hSrcDS) != 3 && GDALGetRasterCount(c->hSrcDS) != 4) { + ctx->set_error(ctx, 500, "Input gdal source for %s has %d raster bands, but only 3 or 4 are supported.\n", + p->gdal->source.name, GDALGetRasterCount(c->hSrcDS) ); + GDALClose(c->hSrcDS); + free(c); return; } - if( GDALGetProjectionRef( hDataset ) != NULL - && strlen(GDALGetProjectionRef( hDataset )) > 0 ) - srcSRS = apr_pstrdup(ctx->pool,GDALGetProjectionRef( hDataset )); - - else if( GDALGetGCPProjection( hDataset ) != NULL - && strlen(GDALGetGCPProjection(hDataset)) > 0 - && GDALGetGCPCount( hDataset ) > 1 ) - srcSRS = apr_pstrdup(ctx->pool,GDALGetGCPProjection( hDataset )); - GDALDriverH hDriver = GDALGetDriverByName( "MEM" ); - GDALDatasetH hDstDS; - /* -------------------------------------------------------------------- */ - /* Create a transformation object from the source to */ - /* destination coordinate system. */ - /* -------------------------------------------------------------------- */ - void *hTransformArg = - GDALCreateGenImgProjTransformer( hDataset, srcSRS, - NULL, dstSRS, - TRUE, 1000.0, 0 ); - if( hTransformArg == NULL ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"gdal failed to create SRS transformation object"); + hDstSRS = OSRNewSpatialReference( NULL ); + if( OSRSetFromUserInput( hDstSRS, p->dst_srs ) == OGRERR_NONE ) { + c->dst_srs_wkt = NULL; + OSRExportToWkt( hDstSRS, &c->dst_srs_wkt ); + } + else { + ctx->set_error(ctx,500,"failed to parse gdal srs %s",p->dst_srs); + GDALClose(c->hSrcDS); + free(c); return; } - /* -------------------------------------------------------------------- */ - /* Get approximate output definition. */ - /* -------------------------------------------------------------------- */ - int nPixels, nLines; - double adfDstGeoTransform[6]; - if( GDALSuggestedWarpOutput( hDataset, - GDALGenImgProjTransform, hTransformArg, - adfDstGeoTransform, &nPixels, &nLines ) - != CE_None ) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"gdal failed to create suggested warp output"); - return; + OSRDestroySpatialReference( hDstSRS ); + *conn_ = c; +} + +void mapcache_source_gdal_connection_destructor(void *conn_) { + gdal_connection *c = (gdal_connection*)conn_; + CPLFree(c->dst_srs_wkt); + GDALClose(c->hSrcDS); + free(c); +} + +static mapcache_pooled_connection* _gdal_get_connection(mapcache_context *ctx, mapcache_source_gdal *gdal, const char *dst_srs, const char *gdal_data) +{ + mapcache_pooled_connection *pc; + gdal_connection_params params; + char *key; + + params.gdal = gdal; + params.dst_srs = dst_srs; + params.gdal_data = gdal_data; + + key = apr_pstrcat(ctx->pool, gdal_data, dst_srs, NULL); + + pc = mapcache_connection_pool_get_connection(ctx,key,mapcache_source_gdal_connection_constructor, + mapcache_source_gdal_connection_destructor, ¶ms); + return pc; +} +#ifdef USE_PRE_GDAL2_METHOD +/* Creates a (virtual) dataset that matches an overview level of the source + dataset. This dataset has references on the source dataset, so it should + be closed before it. +*/ +static GDALDatasetH CreateOverviewVRTDataset(GDALDatasetH hSrcDS, + int iOvrLevel) +{ + double adfSrcGeoTransform[6]; + int iBand; + GDALRasterBandH hFirstBand = GDALGetRasterBand(hSrcDS, 1); + GDALRasterBandH hOvr = GDALGetOverview( hFirstBand, iOvrLevel ); + int nFullResXSize = GDALGetRasterBandXSize(hFirstBand); + int nFullResYSize = GDALGetRasterBandYSize(hFirstBand); + int nVRTXSize = GDALGetRasterBandXSize(hOvr); + int nVRTYSize = GDALGetRasterBandYSize(hOvr); + + VRTDatasetH hVRTDS = VRTCreate( nVRTXSize, nVRTYSize ); + + /* Scale geotransform */ + if( GDALGetGeoTransform( hSrcDS, adfSrcGeoTransform) == CE_None ) + { + adfSrcGeoTransform[1] *= (double)nFullResXSize / nVRTXSize; + adfSrcGeoTransform[2] *= (double)nFullResYSize / nVRTYSize; + adfSrcGeoTransform[4] *= (double)nFullResXSize / nVRTXSize; + adfSrcGeoTransform[5] *= (double)nFullResYSize / nVRTYSize; + GDALSetProjection( hVRTDS, GDALGetProjectionRef(hSrcDS) ); + GDALSetGeoTransform( hVRTDS, adfSrcGeoTransform ); + } + + /* Scale GCPs */ + if( GDALGetGCPCount(hSrcDS) > 0 ) + { + const GDAL_GCP* pasGCPsMain = GDALGetGCPs(hSrcDS); + int nGCPCount = GDALGetGCPCount(hSrcDS); + GDAL_GCP* pasGCPList = GDALDuplicateGCPs( nGCPCount, pasGCPsMain ); + int i; + for(i = 0; i < nGCPCount; i++) + { + pasGCPList[i].dfGCPPixel *= (double)nVRTXSize / nFullResXSize; + pasGCPList[i].dfGCPLine *= (double)nVRTYSize / nFullResYSize; + } + GDALSetGCPs( hVRTDS, nGCPCount, pasGCPList, GDALGetGCPProjection(hSrcDS) ); + GDALDeinitGCPs( nGCPCount, pasGCPList ); + CPLFree( pasGCPList ); + } + + /* Create bands */ + for(iBand = 1; iBand <= GDALGetRasterCount(hSrcDS); iBand++ ) + { + GDALRasterBandH hSrcBand; + GDALRasterBandH hVRTBand; + int bNoDataSet = FALSE; + double dfNoData; + + VRTAddBand( hVRTDS, GDT_Byte, NULL ); + hVRTBand = GDALGetRasterBand(hVRTDS, iBand); + hSrcBand = GDALGetOverview(GDALGetRasterBand(hSrcDS, iBand), iOvrLevel); + dfNoData = GDALGetRasterNoDataValue(hSrcBand, &bNoDataSet); + if( bNoDataSet ) + GDALSetRasterNoDataValue(hVRTBand, dfNoData); + + /* Note: the consumer of this VRT is the warper, which doesn't do any */ + /* subsampled RasterIO requests, so NEAR is fine */ + VRTAddSimpleSource( hVRTBand, hSrcBand, + 0, 0, nVRTXSize, nVRTYSize, + 0, 0, nVRTXSize, nVRTYSize, + "NEAR", VRT_NODATA_UNSET ); } - GDALDestroyGenImgProjTransformer( hTransformArg ); - double dfXRes = (tile->bbox[2] - tile->bbox[0]) / tile->sx; - double dfYRes = (tile->bbox[3] - tile->bbox[1]) / tile->sy; - - adfDstGeoTransform[0] = tile->bbox[0]; - adfDstGeoTransform[3] = tile->bbox[3]; - adfDstGeoTransform[1] = dfXRes; - adfDstGeoTransform[5] = -dfYRes; - hDstDS = GDALCreate( hDriver, "tempd_gdal_image", tile->sx, tile->sy, 4, GDT_Byte, NULL ); + return hVRTDS; +} +#endif + +/* Derived from GDALAutoCreateWarpedVRT(), with various improvements. */ +/* Returns a warped VRT that covers the passed extent, in pszDstWKT. */ +/* The provided width and height are used, but the size of the returned dataset */ +/* may not match those values. In the USE_PRE_GDAL2_METHOD, it should match them. */ +/* In the non USE_PRE_GDAL2_METHOD case, it might be a multiple of those values. */ +/* phTmpDS is an output parameter (a temporary VRT in the USE_PRE_GDAL2_METHOD case). */ +static GDALDatasetH +CreateWarpedVRT( GDALDatasetH hSrcDS, + const char *pszSrcWKT, + const char *pszDstWKT, + int width, int height, + const mapcache_extent *extent, + GDALResampleAlg eResampleAlg, + double dfMaxError, + char** papszWarpOptions, + GDALDatasetH *phTmpDS ) - /* -------------------------------------------------------------------- */ - /* Write out the projection definition. */ - /* -------------------------------------------------------------------- */ - GDALSetProjection( hDstDS, dstSRS ); - GDALSetGeoTransform( hDstDS, adfDstGeoTransform ); - char **papszWarpOptions = NULL; - papszWarpOptions = CSLSetNameValue( papszWarpOptions, "INIT", "0" ); +{ + int i; + GDALWarpOptions *psWO; + double adfDstGeoTransform[6]; + GDALDatasetH hDstDS; + int nDstPixels, nDstLines; + CPLErr eErr; + int bHaveNodata = FALSE; + char** papszOptions = NULL; + +/* -------------------------------------------------------------------- */ +/* Populate the warp options. */ +/* -------------------------------------------------------------------- */ + + psWO = GDALCreateWarpOptions(); + psWO->papszWarpOptions = CSLDuplicate(papszWarpOptions); + + psWO->eResampleAlg = eResampleAlg; + + psWO->hSrcDS = hSrcDS; + + psWO->nBandCount = GDALGetRasterCount( hSrcDS ); + if( psWO->nBandCount == 4 ) + { + psWO->nBandCount = 3; + psWO->nSrcAlphaBand = 4; + psWO->nDstAlphaBand = 4; + } + /* Due to the reprojection, we might get transparency in the edges */ + else if( psWO->nBandCount == 3 ) + psWO->nDstAlphaBand = 4; + + psWO->panSrcBands = (int*)CPLMalloc( sizeof(int) * psWO->nBandCount ); + psWO->panDstBands = (int*)CPLMalloc( sizeof(int) * psWO->nBandCount ); + + for( i = 0; i < psWO->nBandCount; i++ ) + { + psWO->panSrcBands[i] = i+1; + psWO->panDstBands[i] = i+1; + } + +/* -------------------------------------------------------------------- */ +/* Set nodata values if existing */ +/* -------------------------------------------------------------------- */ + GDALGetRasterNoDataValue( GDALGetRasterBand(hSrcDS, 1), &bHaveNodata); + if( bHaveNodata ) + { + psWO->padfSrcNoDataReal = (double *) + CPLMalloc(psWO->nBandCount*sizeof(double)); + psWO->padfSrcNoDataImag = (double *) + CPLCalloc(psWO->nBandCount, sizeof(double)); /* zero initialized */ + + for( i = 0; i < psWO->nBandCount; i++ ) + { + GDALRasterBandH hBand = GDALGetRasterBand( hSrcDS, i+1 ); + + double dfReal = GDALGetRasterNoDataValue( hBand, &bHaveNodata ); + + if( bHaveNodata ) + { + psWO->padfSrcNoDataReal[i] = dfReal; + } + else + { + psWO->padfSrcNoDataReal[i] = -123456.789; + } + } + } +/* -------------------------------------------------------------------- */ +/* Create the transformer. */ +/* -------------------------------------------------------------------- */ + psWO->pfnTransformer = GDALGenImgProjTransform; + psWO->pTransformerArg = + GDALCreateGenImgProjTransformer( psWO->hSrcDS, pszSrcWKT, + NULL, pszDstWKT, + TRUE, 1.0, 0 ); + + if( psWO->pTransformerArg == NULL ) + { + GDALDestroyWarpOptions( psWO ); + return NULL; + } +/* -------------------------------------------------------------------- */ +/* Figure out the desired output bounds and resolution. */ +/* -------------------------------------------------------------------- */ + eErr = + GDALSuggestedWarpOutput( hSrcDS, psWO->pfnTransformer, + psWO->pTransformerArg, + adfDstGeoTransform, &nDstPixels, &nDstLines ); + if( eErr != CE_None ) + { + GDALDestroyTransformer( psWO->pTransformerArg ); + GDALDestroyWarpOptions( psWO ); + return NULL; + } - /* -------------------------------------------------------------------- */ - /* Create a transformation object from the source to */ - /* destination coordinate system. */ - /* -------------------------------------------------------------------- */ - GDALTransformerFunc pfnTransformer = NULL; - void *hGenImgProjArg=NULL, *hApproxArg=NULL; - hTransformArg = hGenImgProjArg = - GDALCreateGenImgProjTransformer( hDataset, srcSRS, - hDstDS, dstSRS, - TRUE, 1000.0, 0 ); - - if( hTransformArg == NULL ) - exit( 1 ); - - pfnTransformer = GDALGenImgProjTransform; - - hTransformArg = hApproxArg = - GDALCreateApproxTransformer( GDALGenImgProjTransform, - hGenImgProjArg, 0.125 ); - pfnTransformer = GDALApproxTransform; +/* -------------------------------------------------------------------- */ +/* To minimize the risk of extra resampling done by generic */ +/* RasterIO itself and maximize resampling done in the wraper, */ +/* adjust the resolution so that the overview factor of the output */ +/* dataset that will indirectly query matches an exiting overview */ +/* factor of the input dataset. */ +/* -------------------------------------------------------------------- */ + { + double dfDesiredXRes = (extent->maxx - extent->minx) / width; + double dfDesiredYRes = (extent->maxy - extent->miny) / height; + double dfDesiredRes = MIN( dfDesiredXRes, dfDesiredYRes ); + double dfGuessedFullRes = MIN( adfDstGeoTransform[1], + fabs(adfDstGeoTransform[5]) ); + double dfApproxDstOvrRatio = dfDesiredRes / dfGuessedFullRes; + + GDALRasterBandH hFirstBand = GDALGetRasterBand(hSrcDS, 1); + int nOvrCount = GDALGetOverviewCount(hFirstBand); + int nSrcXSize = GDALGetRasterBandXSize(hFirstBand); + int i; + double dfSrcOvrRatio = 1.0; +#ifdef USE_PRE_GDAL2_METHOD + int iSelectedOvr = -1; +#endif + for( i = 0; *phTmpDS == NULL && i < nOvrCount; i ++) + { + GDALRasterBandH hOvr = GDALGetOverview(hFirstBand, i); + int nOvrXSize = GDALGetRasterBandXSize(hOvr); + double dfCurOvrRatio = (double)nSrcXSize / nOvrXSize; + if(dfCurOvrRatio > dfApproxDstOvrRatio+0.1 ) /* +0.1 to avoid rounding issues */ + { + break; + } + dfSrcOvrRatio = dfCurOvrRatio; +#ifdef USE_PRE_GDAL2_METHOD + iSelectedOvr = i; +#endif + } + +#ifdef USE_PRE_GDAL2_METHOD + if( iSelectedOvr >= 0 ) + { + GDALDestroyTransformer( psWO->pTransformerArg ); + GDALDestroyWarpOptions( psWO ); + + *phTmpDS = CreateOverviewVRTDataset(hSrcDS, iSelectedOvr); + return CreateWarpedVRT( *phTmpDS, + pszSrcWKT, + pszDstWKT, + width, height, + extent, + eResampleAlg, + dfMaxError, + papszWarpOptions, + phTmpDS ); + } +#endif - /* -------------------------------------------------------------------- */ - /* Now actually invoke the warper to do the work. */ - /* -------------------------------------------------------------------- */ - GDALSimpleImageWarp( hDataset, hDstDS, 0, NULL, - pfnTransformer, hTransformArg, - GDALDummyProgress, NULL, papszWarpOptions ); + adfDstGeoTransform[1] = dfDesiredXRes / dfSrcOvrRatio; + adfDstGeoTransform[5] = -dfDesiredYRes / dfSrcOvrRatio; + } - CSLDestroy( papszWarpOptions ); +/* -------------------------------------------------------------------- */ +/* Compute geotransform and raster dimension for our extent of */ +/* interest. */ +/* -------------------------------------------------------------------- */ + adfDstGeoTransform[0] = extent->minx; + adfDstGeoTransform[2] = 0.0; + adfDstGeoTransform[3] = extent->maxy; + adfDstGeoTransform[4] = 0.0; + nDstPixels = (int)( (extent->maxx - extent->minx) / adfDstGeoTransform[1] + 0.5 ); + nDstLines = (int)( (extent->maxy - extent->miny) / fabs(adfDstGeoTransform[5]) + 0.5 ); + /*printf("nDstPixels=%d nDstLines=%d\n", nDstPixels, nDstLines);*/ + +/* -------------------------------------------------------------------- */ +/* Update the transformer to include an output geotransform */ +/* back to pixel/line coordinates. */ +/* -------------------------------------------------------------------- */ + + GDALSetGenImgProjTransformerDstGeoTransform( + psWO->pTransformerArg, adfDstGeoTransform ); + +/* -------------------------------------------------------------------- */ +/* Do we want to apply an approximating transformation? */ +/* -------------------------------------------------------------------- */ + if( dfMaxError > 0.0 ) + { + psWO->pTransformerArg = + GDALCreateApproxTransformer( psWO->pfnTransformer, + psWO->pTransformerArg, + dfMaxError ); + psWO->pfnTransformer = GDALApproxTransform; + GDALApproxTransformerOwnsSubtransformer(psWO->pTransformerArg, TRUE); + } - if( hApproxArg != NULL ) - GDALDestroyApproxTransformer( hApproxArg ); +/* -------------------------------------------------------------------- */ +/* Create the VRT file. */ +/* -------------------------------------------------------------------- */ + + /* We could potentially used GDALCreateWarpedVRT() instead of this logic */ + /* but GDALCreateWarpedVRT() in GDAL < 2.0.1 doesn't create the destination */ + /* alpha band. */ + + papszOptions = CSLSetNameValue(NULL, "SUBCLASS", "VRTWarpedDataset"); + hDstDS = GDALCreate( GDALGetDriverByName("VRT"), "", nDstPixels, nDstLines, + psWO->nBandCount + ((psWO->nDstAlphaBand != 0) ? 1 : 0), + GDT_Byte, papszOptions ); + CSLDestroy(papszOptions); + if( hDstDS == NULL ) + { + GDALDestroyWarpOptions( psWO ); + return NULL; + } - if( hGenImgProjArg != NULL ) - GDALDestroyGenImgProjTransformer( hGenImgProjArg ); + psWO->hDstDS = hDstDS; - if(GDALGetRasterCount(hDstDS) != 4) { - ctx->set_error(ctx,MAPCACHE_SOURCE_GDAL_ERROR,"gdal did not create a 4 band image"); - return; + GDALSetGeoTransform( hDstDS, adfDstGeoTransform ); + if( GDALInitializeWarpedVRT( hDstDS, psWO ) != CE_None ) + { + GDALClose(hDstDS); + GDALDestroyWarpOptions( psWO ); + return NULL; + } + + GDALDestroyWarpOptions( psWO ); + + return hDstDS; +} + + +/** + * \private \memberof mapcache_source_gdal + * \sa mapcache_source::render_metatile() + */ +void _mapcache_source_gdal_render_metatile(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) +{ + mapcache_source_gdal *gdal = (mapcache_source_gdal*)psource; + gdal_connection *gdal_conn; + GDALDatasetH hDstDS; + GDALDatasetH hTmpDS = NULL; + mapcache_buffer *data; + unsigned char *rasterdata; + CPLErr eErr; + mapcache_pooled_connection *pc = NULL; + int bands_bgra[] = { 3, 2, 1, 4 }; /* mapcache buffer order is BGRA */ + + CPLErrorReset(); + + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + pc = _gdal_get_connection(ctx, gdal, map->grid_link->grid->srs, gdal->datastr ); + GC_CHECK_ERROR(ctx); + gdal_conn = (gdal_connection*) pc->connection; + } else { + gdal_connection_params params; + params.gdal = gdal; + params.dst_srs = map->grid_link->grid->srs; + params.gdal_data = gdal->datastr; + mapcache_source_gdal_connection_constructor(ctx, (void**)(&gdal_conn), ¶ms); + GC_CHECK_ERROR(ctx); } - GDALRasterBandH *redband, *greenband, *blueband, *alphaband; - redband = GDALGetRasterBand(hDstDS,1); - greenband = GDALGetRasterBand(hDstDS,2); - blueband = GDALGetRasterBand(hDstDS,3); - alphaband = GDALGetRasterBand(hDstDS,4); - - unsigned char *rasterdata = apr_palloc(ctx->pool,tile->sx*tile->sy*4); - data->buf = rasterdata; - data->avail = tile->sx*tile->sy*4; - data->size = tile->sx*tile->sy*4; - - GDALRasterIO(redband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - GDALRasterIO(greenband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata+1),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - GDALRasterIO(blueband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata+2),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - if(GDALGetRasterCount(hDataset)==4) - GDALRasterIO(alphaband,GF_Read,0,0,tile->sx,tile->sy,(void*)(rasterdata+3),tile->sx,tile->sy,GDT_Byte,4,4*tile->sx); - else { - unsigned char *alphaptr; - int i; - for(alphaptr = rasterdata+3, i=0; isx*tile->sy; i++, alphaptr+=4) { - *alphaptr = 255; + + hDstDS = CreateWarpedVRT( gdal_conn->hSrcDS, gdal->srs_wkt, gdal_conn->dst_srs_wkt, + map->width, map->height, + &map->extent, + gdal->eResampleAlg, 0.125, NULL, &hTmpDS ); + + if( hDstDS == NULL ) { + ctx->set_error(ctx, 500,"CreateWarpedVRT() failed"); + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); + } + return; + } + + if(GDALGetRasterCount(hDstDS) != 4) { + ctx->set_error(ctx, 500,"gdal did not create a 4 band image"); + GDALClose(hDstDS); /* close first this one, as it references hSrcDS */ + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); } + return; } - tile->imdata = mapcache_image_create(ctx); - tile->imdata->w = tile->sx; - tile->imdata->h = tile->sy; - tile->imdata->stride = tile->sx * 4; - tile->imdata->data = rasterdata; + data = mapcache_buffer_create(map->height*map->width*4,ctx->pool); + rasterdata = data->buf; + +#if GDAL_VERSION_MAJOR >= 2 + { + GDALRasterIOExtraArg sExtraArg; + INIT_RASTERIO_EXTRA_ARG(sExtraArg); + if( gdal->eResampleAlg == GRA_Bilinear ) + sExtraArg.eResampleAlg = GRIORA_Bilinear; + else if( gdal->eResampleAlg == GRA_Cubic ) + sExtraArg.eResampleAlg = GRIORA_Cubic; + else if( gdal->eResampleAlg == GRA_CubicSpline ) + sExtraArg.eResampleAlg = GRIORA_CubicSpline; + else if( gdal->eResampleAlg == GRA_Lanczos ) + sExtraArg.eResampleAlg = GRIORA_Lanczos; + else if( gdal->eResampleAlg == GRA_Average ) + sExtraArg.eResampleAlg = GRIORA_Average; + + if( gdal->srcOvrLevel != NULL ) + { + /* If the user specified a particular strategy to choose the source */ + /* overview level, apply it now */ + GDALSetMetadataItem(hDstDS, "SrcOvrLevel",gdal->srcOvrLevel, NULL); + } + /* Hopefully, given how we adjust hDstDS resolution, we should query */ + /* exactly at the resolution of one overview level of hDstDS, and not */ + /* do extra resampling in generic RasterIO, but just in case specify */ + /* the resampling alg in sExtraArg. */ + eErr = GDALDatasetRasterIOEx( hDstDS, GF_Read,0,0, + GDALGetRasterXSize(hDstDS), + GDALGetRasterYSize(hDstDS), + rasterdata,map->width,map->height,GDT_Byte, + 4, bands_bgra, + 4,4*map->width,1, &sExtraArg ); + } +#else + eErr = GDALDatasetRasterIO( hDstDS, GF_Read,0,0, + GDALGetRasterXSize(hDstDS), + GDALGetRasterYSize(hDstDS), + rasterdata,map->width,map->height,GDT_Byte, + 4, bands_bgra, + 4,4*map->width,1 ); +#endif + + if( eErr != CE_None ) { + ctx->set_error(ctx, 500,"GDAL I/O error occurred"); + GDALClose(hDstDS); /* close first this one, as it references hTmpDS or hSrcDS */ + if( hTmpDS ) + GDALClose(hTmpDS); /* references hSrcDS, so close before */ + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_invalidate_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); + } + return; + } - GDALClose( hDstDS ); - GDALClose( hDataset); + map->raw_image = mapcache_image_create(ctx); + map->raw_image->w = map->width; + map->raw_image->h = map->height; + map->raw_image->stride = map->width * 4; + map->raw_image->data = rasterdata; + map->raw_image->has_alpha = MC_ALPHA_UNKNOWN; + + GDALClose( hDstDS ); /* close first this one, as it references hTmpDS or hSrcDS */ + if( hTmpDS ) + GDALClose(hTmpDS); /* references hSrcDS, so close before */ + if(gdal->bUseConnectionPool == MAPCACHE_TRUE) { + mapcache_connection_pool_release_connection(ctx,pc); + } else { + mapcache_source_gdal_connection_destructor(gdal_conn); + } } /** * \private \memberof mapcache_source_gdal * \sa mapcache_source::configuration_parse() */ -void _mapcache_source_gdal_configuration_parse(mapcache_context *ctx, ezxml_t node, mapcache_source *source) +void _mapcache_source_gdal_configuration_parse(mapcache_context *ctx, ezxml_t node, mapcache_source *source, mapcache_cfg *config) { ezxml_t cur_node; mapcache_source_gdal *src = (mapcache_source_gdal*)source; @@ -228,12 +612,41 @@ if ((cur_node = ezxml_child(node,"data")) != NULL) { src->datastr = apr_pstrdup(ctx->pool,cur_node->txt); } + if ((cur_node = ezxml_child(node,"connection_pooled")) != NULL) { + if(!strcasecmp(cur_node->txt,"false")) { + src->bUseConnectionPool = MAPCACHE_FALSE; + } else if(!strcasecmp(cur_node->txt,"true")) { + src->bUseConnectionPool = MAPCACHE_TRUE; + } else { + ctx->set_error(ctx,400,"failed to parse (%s). Expecting true or false",cur_node->txt); + return; + } + } - if ((cur_node = ezxml_child(node,"gdalparams")) != NULL) { - for(cur_node = cur_node->child; cur_node; cur_node = cur_node->sibling) { - apr_table_set(src->gdal_params, cur_node->name, cur_node->txt); + if ((cur_node = ezxml_child(node,"resample")) != NULL && *cur_node->txt) { + if( EQUALN( cur_node->txt, "NEAR", 4) ) + src->eResampleAlg = GRA_NearestNeighbour; + else if( EQUAL( cur_node->txt, "BILINEAR") ) + src->eResampleAlg = GRA_Bilinear; + else if( EQUAL( cur_node->txt, "CUBIC") ) + src->eResampleAlg = GRA_Cubic; + else if( EQUAL( cur_node->txt, "CUBICSPLINE") ) + src->eResampleAlg = GRA_CubicSpline; + else if( EQUAL( cur_node->txt, "LANCZOS") ) + src->eResampleAlg = GRA_Lanczos; +#if GDAL_VERSION_MAJOR >= 2 + else if( EQUAL( cur_node->txt, "AVERAGE") ) + src->eResampleAlg = GRA_Average; +#endif + else { + ctx->set_error(ctx, 500, "unsupported gdal : %s", cur_node->txt); + return; } } + + if ((cur_node = ezxml_child(node,"overview-strategy")) != NULL && *cur_node->txt) { + src->srcOvrLevel = apr_pstrdup(ctx->pool,cur_node->txt); + } } /** @@ -244,16 +657,31 @@ mapcache_source *source) { mapcache_source_gdal *src = (mapcache_source_gdal*)source; + GDALDatasetH hDataset; + /* check all required parameters are configured */ - if(!strlen(src->datastr)) { - ctx->set_error(ctx, MAPCACHE_SOURCE_GDAL_ERROR, "gdal source %s has no data",source->name); + if( src->datastr == NULL || !strlen(src->datastr)) { + ctx->set_error(ctx, 500, "gdal source %s has no data",source->name); + return; + } + hDataset = GDALOpen(src->datastr,GA_ReadOnly); + if( hDataset == NULL ) { + ctx->set_error(ctx, 500, "gdalOpen failed on data %s", src->datastr); return; } - src->poDataset = (GDALDatasetH*)GDALOpen(src->datastr,GA_ReadOnly); - if( src->poDataset == NULL ) { - ctx->set_error(ctx, MAPCACHE_SOURCE_GDAL_ERROR, "gdalOpen failed on data %s", src->datastr); + if( GDALGetProjectionRef( hDataset ) != NULL + && strlen(GDALGetProjectionRef( hDataset )) > 0 ) { + src->srs_wkt = apr_pstrdup(ctx->pool,GDALGetProjectionRef( hDataset )); + } else if( GDALGetGCPProjection( hDataset ) != NULL + && strlen(GDALGetGCPProjection(hDataset)) > 0 + && GDALGetGCPCount( hDataset ) > 1 ) { + src->srs_wkt = apr_pstrdup(ctx->pool,GDALGetGCPProjection( hDataset )); + } else { + ctx->set_error(ctx, 500, "Input gdal source for %s has no defined SRS\n", source->name ); + GDALClose(hDataset); return; } + GDALClose(hDataset); } #endif //USE_GDAL @@ -261,18 +689,19 @@ mapcache_source* mapcache_source_gdal_create(mapcache_context *ctx) { #ifdef USE_GDAL - GDALAllRegister(); mapcache_source_gdal *source = apr_pcalloc(ctx->pool, sizeof(mapcache_source_gdal)); if(!source) { - ctx->set_error(ctx, MAPCACHE_ALLOC_ERROR, "failed to allocate gdal source"); + ctx->set_error(ctx, 500, "failed to allocate gdal source"); return NULL; } mapcache_source_init(ctx, &(source->source)); source->source.type = MAPCACHE_SOURCE_GDAL; - source->source.render_metatile = _mapcache_source_gdal_render_metatile; + source->source._render_map = _mapcache_source_gdal_render_metatile; source->source.configuration_check = _mapcache_source_gdal_configuration_check; - source->source.configuration_parse = _mapcache_source_gdal_configuration_parse; - source->gdal_params = apr_table_make(ctx->pool,4); + source->source.configuration_parse_xml = _mapcache_source_gdal_configuration_parse; + source->eResampleAlg = MAPCACHE_DEFAULT_RESAMPLE_ALG; + source->bUseConnectionPool = MAPCACHE_TRUE; + GDALAllRegister(); return (mapcache_source*)source; #else ctx->set_error(ctx, 400, "failed to create gdal source, GDAL support is not compiled in this version"); diff -Nru mapcache-1.4.0/lib/source_mapserver.c mapcache-1.6.1/lib/source_mapserver.c --- mapcache-1.4.0/lib/source_mapserver.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/source_mapserver.c 2017-09-29 21:31:13.000000000 +0000 @@ -41,8 +41,15 @@ #include #include -/* hash table key = source->name, value = apr_reslist_t of mapObjs */ -static apr_hash_t *mapobj_container = NULL; +/**\class mapcache_source_mapserver + * \brief WMS mapcache_source + * \implements mapcache_source + */ +typedef struct mapcache_source_mapserver mapcache_source_mapserver; +struct mapcache_source_mapserver { + mapcache_source source; + char *mapfile; +}; struct mc_mapobj { mapObj *map; @@ -50,91 +57,44 @@ char *error; }; -static apr_status_t _ms_get_mapobj(void **conn_, void *params, apr_pool_t *pool) -{ +void mapcache_mapserver_connection_constructor(mapcache_context *ctx, void **conn_, void *params) { mapcache_source_mapserver *src = (mapcache_source_mapserver*) params; struct mc_mapobj *mcmap = calloc(1,sizeof(struct mc_mapobj)); - *conn_ = mcmap; mcmap->map = msLoadMap(src->mapfile,NULL); if(!mcmap->map) { errorObj *errors = NULL; - msWriteError(stderr); + ctx->set_error(ctx, 500, "Failed to load mapfile '%s'",src->mapfile); errors = msGetErrorObj(); - mcmap->error = apr_psprintf(pool,"Failed to load mapfile '%s'. Mapserver reports: %s",src->mapfile, errors->message); - return APR_EGENERAL; + while(errors) { + ctx->set_error(ctx, 500, "Failed to load mapfile '%s'. Mapserver reports: %s",src->mapfile, errors->message); + errors = errors->next; + } + return; } msMapSetLayerProjections(mcmap->map); - return APR_SUCCESS; + *conn_ = mcmap; } -static apr_status_t _ms_free_mapobj(void *conn_, void *params, apr_pool_t *pool) -{ +void mapcache_mapserver_connection_destructor(void *conn_) { struct mc_mapobj *mcmap = (struct mc_mapobj*) conn_; msFreeMap(mcmap->map); free(mcmap); - return APR_SUCCESS; -} - -static struct mc_mapobj* _get_mapboj(mapcache_context *ctx, mapcache_map *map) { - apr_status_t rv; - mapcache_source_mapserver *src = (mapcache_source_mapserver*) map->tileset->source; - struct mc_mapobj *mcmap; - apr_reslist_t *mapobjs = NULL; - if(!mapobj_container || NULL == (mapobjs = apr_hash_get(mapobj_container,src->source.name,APR_HASH_KEY_STRING))) { -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_lock((apr_thread_mutex_t*)ctx->threadlock); -#endif - if(!mapobj_container) { - mapobj_container = apr_hash_make(ctx->process_pool); - } - mapobjs = apr_hash_get(mapobj_container,src->source.name,APR_HASH_KEY_STRING); - if(!mapobjs) { - apr_status_t rv; - rv = apr_reslist_create(&mapobjs, - 0 /* min */, - 1 /* soft max */, - 30 /* hard max */, - 6 * 1000000 /*6 seconds, ttl*/, - _ms_get_mapobj, /* resource constructor */ - _ms_free_mapobj, /* resource destructor */ - src, ctx->process_pool); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to create mapobj connection pool for cache %s", src->source.name); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - return NULL; - } - apr_hash_set(mapobj_container,src->source.name,APR_HASH_KEY_STRING,mapobjs); - } - assert(mapobjs); -#ifdef APR_HAS_THREADS - if(ctx->threadlock) - apr_thread_mutex_unlock((apr_thread_mutex_t*)ctx->threadlock); -#endif - } - rv = apr_reslist_acquire(mapobjs, (void **) &mcmap); - if (rv != APR_SUCCESS) { - ctx->set_error(ctx, 500, "failed to aquire mappObj instance: %s", mcmap->error); - return NULL; - } - return mcmap; } -static void _release_mapboj(mapcache_context *ctx, mapcache_map *map, struct mc_mapobj *mcmap) +static mapcache_pooled_connection* _mapserver_get_connection(mapcache_context *ctx, mapcache_map *map) { - mapcache_source_mapserver *src = (mapcache_source_mapserver*) map->tileset->source; - msFreeLabelCache(&mcmap->map->labelcache); - apr_reslist_t *mapobjs = apr_hash_get(mapobj_container,src->source.name, APR_HASH_KEY_STRING); - assert(mapobjs); - if (GC_HAS_ERROR(ctx)) { - apr_reslist_invalidate(mapobjs, (void*) mcmap); - } else { - apr_reslist_release(mapobjs, (void*) mcmap); + mapcache_pooled_connection *pc; + char *key = apr_psprintf(ctx->pool, "ms_src_%s", map->tileset->source->name); + + pc = mapcache_connection_pool_get_connection(ctx, key, mapcache_mapserver_connection_constructor, + mapcache_mapserver_connection_destructor, map->tileset->source); + if(!GC_HAS_ERROR(ctx) && pc && pc->connection) { } + + return pc; } + + /** * \private \memberof mapcache_source_mapserver * \sa mapcache_source::render_map() @@ -142,15 +102,27 @@ void _mapcache_source_mapserver_render_map(mapcache_context *ctx, mapcache_map *map) { errorObj *errors = NULL; + mapcache_pooled_connection *pc; + struct mc_mapobj *mcmap; + double dx, dy; + rasterBufferObj rb; + imageObj *image; - struct mc_mapobj *mcmap = _get_mapboj(ctx,map); + pc = _mapserver_get_connection(ctx, map); + GC_CHECK_ERROR(ctx); + + mcmap = pc->connection; GC_CHECK_ERROR(ctx); if(mcmap->grid_link != map->grid_link) { if (msLoadProjectionString(&(mcmap->map->projection), map->grid_link->grid->srs) != 0) { + ctx->set_error(ctx,500, "Unable to set projection on mapObj."); errors = msGetErrorObj(); - ctx->set_error(ctx,500, "Unable to set projection on mapObj. MapServer reports: %s", errors->message); - _release_mapboj(ctx,map,mcmap); + while(errors) { + ctx->set_error(ctx,500, "Unable to set projection on mapObj. MapServer reports: %s", errors->message); + errors = errors->next; + } + mapcache_connection_pool_invalidate_connection(ctx,pc); return; } switch(map->grid_link->grid->unit) { @@ -173,7 +145,6 @@ ** pixel to center of pixel. Here we try to adjust the WMS extents ** in by half a pixel. */ - double dx, dy; dx = (map->extent.maxx - map->extent.minx) / (map->width*2); dy = (map->extent.maxy - map->extent.miny) / (map->height*2); @@ -183,20 +154,27 @@ mcmap->map->extent.maxy = map->extent.maxy - dy; msMapSetSize(mcmap->map, map->width, map->height); - imageObj *image = msDrawMap(mcmap->map, MS_FALSE); + image = msDrawMap(mcmap->map, MS_FALSE); if(!image) { + ctx->set_error(ctx,500, "MapServer failed to create image."); errors = msGetErrorObj(); - ctx->set_error(ctx,500, "MapServer failed to create image. MapServer reports: %s", errors->message); - _release_mapboj(ctx,map,mcmap); + while(errors) { + ctx->set_error(ctx,500, "MapServer reports: %s", errors->message); + errors = errors->next; + } + mapcache_connection_pool_invalidate_connection(ctx,pc); return; } - rasterBufferObj rb; if(image->format->vtable->supports_pixel_buffer) { - image->format->vtable->getRasterBufferHandle(image,&rb); + if( MS_SUCCESS != image->format->vtable->getRasterBufferHandle(image,&rb)) { + ctx->set_error(ctx,500,"failed to get mapserver raster buffer handle"); + mapcache_connection_pool_invalidate_connection(ctx,pc); + return; + } } else { ctx->set_error(ctx,500,"format %s has no pixel export",image->format->name); - _release_mapboj(ctx,map,mcmap); + mapcache_connection_pool_invalidate_connection(ctx,pc); return; } @@ -208,11 +186,11 @@ memcpy(map->raw_image->data,rb.data.rgba.pixels,map->width*map->height*4); apr_pool_cleanup_register(ctx->pool, map->raw_image->data,(void*)free, apr_pool_cleanup_null); msFreeImage(image); - _release_mapboj(ctx,map,mcmap); + mapcache_connection_pool_release_connection(ctx,pc); } -void _mapcache_source_mapserver_query(mapcache_context *ctx, mapcache_feature_info *fi) +void _mapcache_source_mapserver_query(mapcache_context *ctx, mapcache_source *psource, mapcache_feature_info *fi) { ctx->set_error(ctx,500,"mapserver source does not support queries"); } @@ -238,6 +216,7 @@ mapcache_source *source) { mapcache_source_mapserver *src = (mapcache_source_mapserver*)source; + mapObj *map; /* check all required parameters are configured */ if(!src->mapfile) { ctx->set_error(ctx, 400, "mapserver source %s has no configured",source->name); @@ -250,7 +229,7 @@ msSetup(); /* do a test load to check the mapfile is correct */ - mapObj *map = msLoadMap(src->mapfile, NULL); + map = msLoadMap(src->mapfile, NULL); if(!map) { msWriteError(stderr); ctx->set_error(ctx,400,"failed to load mapfile \"%s\"",src->mapfile); diff -Nru mapcache-1.4.0/lib/source_wms.c mapcache-1.6.1/lib/source_wms.c --- mapcache-1.4.0/lib/source_wms.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/source_wms.c 2017-09-29 21:31:13.000000000 +0000 @@ -33,13 +33,27 @@ #include #include +typedef struct mapcache_source_wms mapcache_source_wms; + +/**\class mapcache_source_wms + * \brief WMS mapcache_source + * \implements mapcache_source + */ +struct mapcache_source_wms { + mapcache_source source; + apr_table_t *wms_default_params; /**< default WMS parameters (SERVICE,REQUEST,STYLES,VERSION) */ + apr_table_t *getmap_params; /**< WMS parameters specified in configuration */ + apr_table_t *getfeatureinfo_params; /**< WMS parameters specified in configuration */ + mapcache_http *http; +}; + /** * \private \memberof mapcache_source_wms * \sa mapcache_source::render_map() */ -void _mapcache_source_wms_render_map(mapcache_context *ctx, mapcache_map *map) +void _mapcache_source_wms_render_map(mapcache_context *ctx, mapcache_source *psource, mapcache_map *map) { - mapcache_source_wms *wms = (mapcache_source_wms*)map->tileset->source; + mapcache_source_wms *wms = (mapcache_source_wms*)psource; mapcache_http *http; apr_table_t *params = apr_table_clone(ctx->pool,wms->wms_default_params); apr_table_setn(params,"BBOX",apr_psprintf(ctx->pool,"%f,%f,%f,%f", @@ -50,21 +64,20 @@ apr_table_setn(params,"SRS",map->grid_link->grid->srs); apr_table_overlap(params,wms->getmap_params,APR_OVERLAP_TABLES_SET); - if(map->dimensions && !apr_is_empty_table(map->dimensions)) { - const apr_array_header_t *elts = apr_table_elts(map->dimensions); + + if(map->dimensions && map->dimensions->nelts>0) { int i; - for(i=0; inelts; i++) { - apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(map->dimensions,i,mapcache_requested_dimension*); /* set both DIM_key=val and key=val KVP params */ - apr_table_setn(params,entry.key,entry.val); - if(strcasecmp(entry.key,"TIME") && strcasecmp(entry.key,"ELEVATION")) { - char *dim_name = apr_pstrcat(ctx->pool,"DIM_",entry.key,NULL); - apr_table_setn(params,dim_name,entry.val); + apr_table_setn(params,rdim->dimension->name,rdim->cached_value); + if(strcasecmp(rdim->dimension->name,"TIME") && strcasecmp(rdim->dimension->name,"ELEVATION")) { + char *dim_name = apr_pstrcat(ctx->pool,"DIM_",rdim->dimension->name,NULL); + apr_table_setn(params,dim_name,rdim->cached_value); } } - } - + /* if the source has no LAYERS parameter defined, then use the tileset name * as the LAYERS to request. When using mirror-mode, the source has no layers * defined, it is added based on the incoming request @@ -86,11 +99,11 @@ } } -void _mapcache_source_wms_query(mapcache_context *ctx, mapcache_feature_info *fi) +void _mapcache_source_wms_query(mapcache_context *ctx, mapcache_source *source, mapcache_feature_info *fi) { mapcache_map *map = (mapcache_map*)fi; mapcache_http *http; - mapcache_source_wms *wms = (mapcache_source_wms*)map->tileset->source; + mapcache_source_wms *wms = (mapcache_source_wms*)source; apr_table_t *params = apr_table_clone(ctx->pool,wms->wms_default_params); apr_table_overlap(params,wms->getmap_params,0); @@ -105,14 +118,18 @@ apr_table_setn(params,"INFO_FORMAT",fi->format); apr_table_overlap(params,wms->getfeatureinfo_params,0); - if(map->dimensions && !apr_is_empty_table(map->dimensions)) { - const apr_array_header_t *elts = apr_table_elts(map->dimensions); + + if(map->dimensions && map->dimensions->nelts>0) { int i; - for(i=0; inelts; i++) { - apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); - apr_table_setn(params,entry.key,entry.val); + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(map->dimensions,i,mapcache_requested_dimension*); + /* set both DIM_key=val and key=val KVP params */ + apr_table_setn(params,rdim->dimension->name,rdim->cached_value); + if(strcasecmp(rdim->dimension->name,"TIME") && strcasecmp(rdim->dimension->name,"ELEVATION")) { + char *dim_name = apr_pstrcat(ctx->pool,"DIM_",rdim->dimension->name,NULL); + apr_table_setn(params,dim_name,rdim->cached_value); + } } - } fi->data = mapcache_buffer_create(30000,ctx->pool); @@ -127,7 +144,7 @@ * \private \memberof mapcache_source_wms * \sa mapcache_source::configuration_parse() */ -void _mapcache_source_wms_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source) +void _mapcache_source_wms_configuration_parse_xml(mapcache_context *ctx, ezxml_t node, mapcache_source *source, mapcache_cfg *config) { ezxml_t cur_node; mapcache_source_wms *src = (mapcache_source_wms*)source; @@ -210,10 +227,10 @@ } mapcache_source_init(ctx, &(source->source)); source->source.type = MAPCACHE_SOURCE_WMS; - source->source.render_map = _mapcache_source_wms_render_map; + source->source._render_map = _mapcache_source_wms_render_map; source->source.configuration_check = _mapcache_source_wms_configuration_check; source->source.configuration_parse_xml = _mapcache_source_wms_configuration_parse_xml; - source->source.query_info = _mapcache_source_wms_query; + source->source._query_info = _mapcache_source_wms_query; source->wms_default_params = apr_table_make(ctx->pool,4);; source->getmap_params = apr_table_make(ctx->pool,4); source->getfeatureinfo_params = apr_table_make(ctx->pool,4); diff -Nru mapcache-1.4.0/lib/strptime.c mapcache-1.6.1/lib/strptime.c --- mapcache-1.4.0/lib/strptime.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/strptime.c 2017-09-29 21:31:13.000000000 +0000 @@ -40,7 +40,6 @@ #include #include "util.h" - static const char *abb_weekdays[] = { "Sun", "Mon", diff -Nru mapcache-1.4.0/lib/tileset.c mapcache-1.6.1/lib/tileset.c --- mapcache-1.4.0/lib/tileset.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/tileset.c 2017-09-29 21:31:13.000000000 +0000 @@ -37,24 +37,22 @@ #include #endif -char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt) -{ +char* mapcache_tileset_tile_resource_key(mapcache_context *ctx, mapcache_tile *tile) { char *lockname = apr_psprintf(ctx->pool, "%d-%d-%d-%s", - mt->z,mt->y,mt->x, - mt->map.tileset->name); + tile->z,tile->y/tile->tileset->metasize_y,tile->x/tile->tileset->metasize_x, + tile->tileset->name); /* if the tileset has multiple grids, add the name of the current grid to the lock key*/ - if(mt->map.tileset->grid_links->nelts > 1) { - lockname = apr_pstrcat(ctx->pool,lockname,mt->map.grid_link->grid->name,NULL); + if(tile->tileset->grid_links->nelts > 1) { + lockname = apr_pstrcat(ctx->pool,lockname,tile->grid_link->grid->name,NULL); } - if(mt->map.dimensions && !apr_is_empty_table(mt->map.dimensions)) { - const apr_array_header_t *elts = apr_table_elts(mt->map.dimensions); + if(tile->dimensions && tile->dimensions->nelts>0) { int i; - for(i=0; inelts; i++) { - apr_table_entry_t entry = APR_ARRAY_IDX(elts,i,apr_table_entry_t); - char *dimvalue = apr_pstrdup(ctx->pool,entry.val); + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + char *dimvalue = apr_pstrdup(ctx->pool,rdim->cached_value); char *iter = dimvalue; while(*iter) { if(*iter == '/') *iter='_'; @@ -67,6 +65,11 @@ return lockname; } +char* mapcache_tileset_metatile_resource_key(mapcache_context *ctx, mapcache_metatile *mt) +{ + return mapcache_tileset_tile_resource_key(ctx,&mt->tiles[0]); +} + void mapcache_tileset_configuration_check(mapcache_context *ctx, mapcache_tileset *tileset) { @@ -151,6 +154,30 @@ tileset->watermark = mapcache_imageio_decode(ctx,watermarkdata); } +void mapcache_tileset_tile_validate_z(mapcache_context *ctx, mapcache_tile *tile) { + if(tile->z < tile->grid_link->minz || tile->z >= tile->grid_link->maxz) { + ctx->set_error(ctx,404,"invalid tile z level"); + } +} + +void mapcache_tileset_tile_validate_x(mapcache_context *ctx, mapcache_tile *tile) { + mapcache_extent_i limits; + limits = tile->grid_link->grid_limits[tile->z]; + if(tile->xx>=limits.maxx) { + ctx->set_error(ctx, 404, "tile x=%d not in [%d,%d[", + tile->x,limits.minx,limits.maxx); + } +} + +void mapcache_tileset_tile_validate_y(mapcache_context *ctx, mapcache_tile *tile) { + mapcache_extent_i limits; + limits = tile->grid_link->grid_limits[tile->z]; + if(tile->yy>=limits.maxy) { + ctx->set_error(ctx, 404, "tile y=%d not in [%d,%d[", + tile->y,limits.miny,limits.maxy); + } +} + void mapcache_tileset_tile_validate(mapcache_context *ctx, mapcache_tile *tile) { mapcache_extent_i limits; @@ -303,6 +330,9 @@ ox = (Mx - tile->x) * tile->grid_link->grid->tile_sx; oy = (tile->y - my) * tile->grid_link->grid->tile_sy; break; + default: + ctx->set_error(ctx,500,"BUG: invalid grid origin"); + return NULL; } if(tile->nodata) continue; @@ -327,7 +357,7 @@ /* copy/scale the srcimage onto the destination image */ tileresolution = toplefttile->grid_link->grid->levels[toplefttile->z]->resolution; - mapcache_grid_get_extent(ctx,toplefttile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx,toplefttile->grid_link->grid, toplefttile->x, toplefttile->y, toplefttile->z, &tilebbox); /*compute the pixel position of top left corner*/ @@ -406,7 +436,7 @@ /* configured metatile size in geographical units */ fullgwidth = res * tileset->metasize_x * grid->tile_sx; fullgheight = res * tileset->metasize_y * grid->tile_sy; - + switch(grid->origin) { case MAPCACHE_GRID_ORIGIN_BOTTOM_LEFT: mt->map.extent.minx = grid->extent.minx + mt->x * fullgwidth - gbuffer; @@ -449,26 +479,16 @@ */ void mapcache_tileset_render_metatile(mapcache_context *ctx, mapcache_metatile *mt) { - int i; -#ifdef DEBUG - if(!mt->map.tileset->source || mt->map.tileset->read_only) { - ctx->set_error(ctx,500,"###BUG### tileset_render_metatile called on tileset with no source or that is read-only"); + mapcache_tileset *tileset = mt->map.tileset; + if(!tileset->source || tileset->read_only) { + ctx->set_error(ctx,500,"tileset_render_metatile called on tileset with no source or that is read-only"); return; } -#endif - mt->map.tileset->source->render_map(ctx, &mt->map); + mapcache_source_render_map(ctx, tileset->source, &mt->map); GC_CHECK_ERROR(ctx); mapcache_image_metatile_split(ctx, mt); GC_CHECK_ERROR(ctx); - if(mt->map.tileset->_cache->tile_multi_set) { - mt->map.tileset->_cache->tile_multi_set(ctx, mt->map.tileset->_cache, mt->tiles, mt->ntiles); - } else { - for(i=0; intiles; i++) { - mapcache_tile *tile = &(mt->tiles[i]); - mt->map.tileset->_cache->tile_set(ctx, mt->map.tileset->_cache, tile); - GC_CHECK_ERROR(ctx); - } - } + mapcache_cache_tile_multi_set(ctx, tileset->_cache, mt->tiles, mt->ntiles); } @@ -488,6 +508,9 @@ tileset->format = NULL; tileset->grid_links = NULL; tileset->config = NULL; + tileset->store_dimension_assemblies = 1; + tileset->dimension_assembly_type = MAPCACHE_DIMENSION_ASSEMBLY_NONE; + tileset->subdimension_read_only = 0; return tileset; } @@ -510,6 +533,9 @@ dst->watermark = src->watermark; dst->wgs84bbox = src->wgs84bbox; dst->format = src->format; + dst->store_dimension_assemblies = src->store_dimension_assemblies; + dst->dimension_assembly_type = src->dimension_assembly_type; + dst->subdimension_read_only = src->subdimension_read_only; return dst; } @@ -528,17 +554,15 @@ tile->grid_link = grid_link; if(tileset->dimensions) { int i; - tile->dimensions = apr_table_make(pool,tileset->dimensions->nelts); + tile->dimensions = apr_array_make(pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); for(i=0; idimensions->nelts; i++) { mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - apr_table_set(tile->dimensions,dimension->name,dimension->default_value); - } - } - if(tileset->timedimension) { - if(!tile->dimensions) { - tile->dimensions = apr_table_make(pool,1); + mapcache_requested_dimension *rdim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + rdim->requested_value = dimension->default_value; + rdim->cached_value = NULL; + rdim->dimension = dimension; + APR_ARRAY_PUSH(tile->dimensions,mapcache_requested_dimension*) = rdim; } - apr_table_set(tile->dimensions,tileset->timedimension->key,tileset->timedimension->default_value); } return tile; } @@ -549,9 +573,7 @@ tile->tileset = src->tileset; tile->expires = src->expires; tile->grid_link = src->grid_link; - if(src->dimensions) { - tile->dimensions = apr_table_clone(pool,src->dimensions); - } + tile->dimensions = mapcache_requested_dimensions_clone(pool, src->dimensions); tile->x = src->x; tile->y = src->y; tile->z = src->z; @@ -565,7 +587,7 @@ map->tileset = src->tileset; map->expires = src->expires; map->grid_link = src->grid_link; - map->dimensions = apr_table_clone(pool,src->dimensions); + map->dimensions = mapcache_requested_dimensions_clone(pool, src->dimensions); map->height = src->height; map->width = src->width; map->extent = src->extent; @@ -582,18 +604,16 @@ map->grid_link = grid_link; if(tileset->dimensions) { int i; - map->dimensions = apr_table_make(pool,tileset->dimensions->nelts); + map->dimensions = apr_array_make(pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); for(i=0; idimensions->nelts; i++) { mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - apr_table_set(map->dimensions,dimension->name,dimension->default_value); + mapcache_requested_dimension *rdim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + rdim->requested_value = dimension->default_value; + rdim->cached_value = NULL; + rdim->dimension = dimension; + APR_ARRAY_PUSH(map->dimensions,mapcache_requested_dimension*) = rdim; } } - if(tileset->timedimension) { - if(!map->dimensions) { - map->dimensions = apr_table_make(pool,1); - } - apr_table_set(map->dimensions,tileset->timedimension->key,tileset->timedimension->default_value); - } return map; } @@ -608,30 +628,35 @@ fi->map.grid_link = grid_link; if(tileset->dimensions) { int i; - fi->map.dimensions = apr_table_make(pool,tileset->dimensions->nelts); + fi->map.dimensions = apr_array_make(pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); for(i=0; idimensions->nelts; i++) { mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - apr_table_set(fi->map.dimensions,dimension->name,dimension->default_value); + mapcache_requested_dimension *rdim = apr_pcalloc(pool,sizeof(mapcache_requested_dimension)); + rdim->requested_value = dimension->default_value; + rdim->cached_value = NULL; + rdim->dimension = dimension; + APR_ARRAY_PUSH(fi->map.dimensions,mapcache_requested_dimension*) = rdim; } } return fi; } void mapcache_tileset_assemble_out_of_zoom_tile(mapcache_context *ctx, mapcache_tile *tile) { - assert(tile->grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE); - - /* we have at most 4 tiles composing the requested tile */ mapcache_extent tile_bbox; double shrink_x, shrink_y, scalefactor; int x[4],y[4]; int i, n=1; - mapcache_grid_get_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z, &tile_bbox); + mapcache_tile *childtile; + assert(tile->grid_link->outofzoom_strategy == MAPCACHE_OUTOFZOOM_REASSEMBLE); + + /* we have at most 4 tiles composing the requested tile */ + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z, &tile_bbox); /* shrink the extent so we do not fall exactly on a tile boundary, to avoid rounding errors when computing the x,y of the lower level tile(s) we will need */ - + shrink_x = (tile_bbox.maxx - tile_bbox.minx) / (tile->grid_link->grid->tile_sx * 1000); /* 1/1000th of a pixel */ shrink_y = (tile_bbox.maxy - tile_bbox.miny) / (tile->grid_link->grid->tile_sy * 1000); /* 1/1000th of a pixel */ tile_bbox.maxx -= shrink_x; @@ -656,15 +681,15 @@ tile_bbox.minx -= shrink_x; tile_bbox.miny -= shrink_y; - mapcache_tile *childtile = mapcache_tileset_tile_clone(ctx->pool,tile); + childtile = mapcache_tileset_tile_clone(ctx->pool,tile); childtile->z = tile->grid_link->max_cached_zoom; scalefactor = childtile->grid_link->grid->levels[childtile->z]->resolution/tile->grid_link->grid->levels[tile->z]->resolution; tile->nodata = 1; for(i=0;ix = x[i]; - childtile->y = y[i]; mapcache_extent childtile_bbox; double dstminx,dstminy; + childtile->x = x[i]; + childtile->y = y[i]; mapcache_tileset_tile_get(ctx,childtile); GC_CHECK_ERROR(ctx); if(childtile->nodata) { @@ -683,7 +708,7 @@ tile->nodata = 0; } /* now copy/scale the srcimage onto the destination image */ - mapcache_grid_get_extent(ctx,childtile->grid_link->grid, + mapcache_grid_get_tile_extent(ctx,childtile->grid_link->grid, childtile->x, childtile->y, childtile->z, &childtile_bbox); /*compute the pixel position of top left corner*/ @@ -700,13 +725,14 @@ * https://bugs.freedesktop.org/show_bug.cgi?id=46277 */ unsigned int row,col; unsigned char *srcpixptr; + unsigned char *row_ptr; unsigned int dstminxi = - dstminx / scalefactor; unsigned int dstminyi = - dstminy / scalefactor; srcpixptr = &(childtile->raw_image->data[dstminyi * childtile->raw_image->stride + dstminxi * 4]); /* ctx->log(ctx, MAPCACHE_WARN, "factor: %g. pixel: %d,%d (val:%d)",scalefactor,dstminxi,dstminyi,*((unsigned int*)srcpixptr)); */ - unsigned char *row_ptr = tile->raw_image->data; + row_ptr = tile->raw_image->data; for(row=0;rowraw_image->h;row++) { unsigned char *pix_ptr = row_ptr; for(col=0;colraw_image->w;col++) { @@ -723,11 +749,6 @@ childtile->raw_image = NULL; childtile->encoded_data = NULL; } - - - - - } void mapcache_tileset_outofzoom_get(mapcache_context *ctx, mapcache_tile *tile) { @@ -743,6 +764,228 @@ } } +int mapcache_tileset_tile_get_readonly(mapcache_context *ctx, mapcache_tile *tile) { + int ret = mapcache_cache_tile_get(ctx, tile->tileset->_cache, tile); + if(GC_HAS_ERROR(ctx)) + return ret; + + if(ret == MAPCACHE_SUCCESS && tile->tileset->auto_expire && tile->mtime && tile->tileset->source && !tile->tileset->read_only) { + /* the cache is in auto-expire mode, and can return the tile modification date, + * and there is a source configured so we can possibly update it, + * so we check to see if it is stale */ + apr_time_t now = apr_time_now(); + apr_time_t stale = tile->mtime + apr_time_from_sec(tile->tileset->auto_expire); + if(staleget_error(ctx) == 404) { + ctx->clear_errors(ctx); + } + ret = MAPCACHE_CACHE_MISS; + } + } + return ret; +} + +typedef struct { + mapcache_tile *tile; + int cache_status; +} mapcache_subtile; + +static void mapcache_tileset_tile_get_without_subdimensions(mapcache_context *ctx, mapcache_tile *tile, int read_only); + +void mapcache_tileset_tile_set_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile) { + apr_array_header_t *subtiles; + mapcache_extent extent; + mapcache_subtile st; + mapcache_image *assembled_image = NULL; + mapcache_buffer *assembled_buffer = NULL; + int i,j,k,n_subtiles = 1,assembled_nodata = 1; + /* we can be here in two cases: + * - either we didn't look up the tile directly (need to split dimension into sub-dimension and reassemble dynamically) + * - either the direct lookup failed and we need to render/assemble the tiles from subdimensions + */ + subtiles = apr_array_make(ctx->pool,1,sizeof(mapcache_subtile)); + st.tile = tile; + APR_ARRAY_PUSH(subtiles,mapcache_subtile) = st; + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + if(GC_HAS_ERROR(ctx)) goto cleanup; + + for(i=0;idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + apr_array_header_t *single_subdimension = rdim->dimension->get_entries_for_value(ctx,rdim->dimension,rdim->requested_value, + tile->tileset, &extent, tile->grid_link->grid); + if(GC_HAS_ERROR(ctx)) /* invalid dimension given */ + goto cleanup; +#ifdef DEBUG + { + char *dims = ""; + int i; + for(i=0;inelts;i++) + dims = apr_pstrcat(ctx->pool,dims,APR_ARRAY_IDX(single_subdimension,i,char*)," ",NULL); + ctx->log(ctx,MAPCACHE_DEBUG,"tile (%d,%d,%d) dimension (%s) returned: %s", + tile->z,tile->y,tile->x,rdim->dimension->name,dims); + } +#endif + + if(single_subdimension->nelts == 0) { + /* not an error, but no subdimension was found: we need to return an empty tile */ + tile->nodata = 1; + if(tile->tileset->store_dimension_assemblies) { + tile->raw_image = mapcache_image_create_with_data(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy); + tile->raw_image->has_alpha = MC_ALPHA_YES; + tile->raw_image->is_blank = MC_EMPTY_YES; + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + /* set the key for the dimension so it can be stored with the requested dimension */ + for(j=0;jdimensions->nelts;j++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,j,mapcache_requested_dimension*); + dim->cached_value = dim->requested_value; + } + mapcache_cache_tile_set(ctx, tile->tileset->_cache, tile); + GC_CHECK_ERROR(ctx); + } + return; + } else { + for(j=0;jnelts;k++) { + st.tile = mapcache_tileset_tile_clone(ctx->pool,APR_ARRAY_IDX(subtiles,j,mapcache_subtile).tile); + APR_ARRAY_PUSH(subtiles,mapcache_subtile)=st; + } + } + n_subtiles *= single_subdimension->nelts; + /* foreach of the subtiles, now set the actual subdimension we are going to be using + the "j%nelts" part takes care of looping over and over through the individual subdimensions */ + for(j=0;jdimension->name, + APR_ARRAY_IDX(single_subdimension,j%single_subdimension->nelts,char*)); + + } + } + } + + /* our subtiles array now contains a list of tiles with subdimensions split up, we now need to fetch them from the cache */ + /* note that subtiles[0].tile == tile */ + + for(i=subtiles->nelts-1; i>=0; i--) { + mapcache_tile *subtile = APR_ARRAY_IDX(subtiles,i,mapcache_subtile).tile; + mapcache_tileset_tile_get_without_subdimensions(ctx, subtile, (tile->tileset->subdimension_read_only||!tile->tileset->source)?1:0); /* creates the tile from the source, takes care of metatiling */ + if(GC_HAS_ERROR(ctx)) + goto cleanup; + if(!subtile->nodata) { + assembled_nodata = 0; + if(!assembled_buffer && !assembled_image) { + /* first "usable" subtile */ + assembled_buffer = subtile->encoded_data; + assembled_image = subtile->raw_image; + } else { + /* need to merge current assembled tile over this subtile */ + if(!assembled_image) { + assembled_image = mapcache_imageio_decode(ctx,assembled_buffer); + if(GC_HAS_ERROR(ctx)) + goto cleanup; + assembled_buffer = NULL; /* the image data went stale as we're merging something */ + } + if(!subtile->raw_image) { + subtile->raw_image = mapcache_imageio_decode(ctx,subtile->encoded_data); + if(GC_HAS_ERROR(ctx)) + goto cleanup; + } + mapcache_image_merge(ctx, subtile->raw_image, assembled_image); + assembled_image = subtile->raw_image; + assembled_image->has_alpha = MC_ALPHA_UNKNOWN; /* we've merged two images, we now have no idea if it's transparent or not */ + if(GC_HAS_ERROR(ctx)) + goto cleanup; + } + if((subtile->encoded_data && mapcache_imageio_header_sniff(ctx,subtile->encoded_data) == GC_JPEG)|| + (subtile->raw_image && subtile->raw_image->has_alpha == MC_ALPHA_NO)) { + /* the returned image is fully opaque, we don't need to get/decode/merge any further subtiles */ + if(assembled_image) + assembled_image->has_alpha = MC_ALPHA_NO; + break; + } + } + } + + tile->encoded_data = assembled_buffer; + tile->raw_image = assembled_image; + tile->nodata = assembled_nodata; + + /* TODO: how should the no data case be handled generically? + * uncomment the following block if this nodata state should be returned to + * the requester immediately, without this info being stored to the cache. + * Leaving this uncommented will cause a no-data tile to be (maybe, depending + * on the cache's actual configuration) written to the cache + */ + /* + if(tile->nodata) { + goto cleanup; + } + */ + + if(!tile->nodata && !tile->encoded_data) { + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + if(tile->tileset->store_dimension_assemblies) { + int already_stored = 1; /*depending on the type of dimension, we may have no nead to store the resulting tile*/ + + if(n_subtiles != 1) + already_stored = 0; /*if we had to merge multiple subdimensions, then we always have to store the resulting assembly*/ + + /* set the key for the dimension so it can be stored with the requested dimension */ + for(j=0;jdimensions->nelts;j++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,j,mapcache_requested_dimension*); + if(strcmp(dim->cached_value,dim->requested_value)) { + already_stored = 0; /*the subdimension is different than the requested dimension, we need to store the resulting tile*/ + } + dim->cached_value = dim->requested_value; + } + if(!already_stored) { + if(tile->nodata) { + tile->raw_image = mapcache_image_create_with_data(ctx,tile->grid_link->grid->tile_sx, tile->grid_link->grid->tile_sy); + tile->raw_image->has_alpha = MC_ALPHA_YES; + tile->raw_image->is_blank = MC_EMPTY_YES; + tile->encoded_data = tile->tileset->format->write(ctx, tile->raw_image, tile->tileset->format); + GC_CHECK_ERROR(ctx); + } + mapcache_cache_tile_set(ctx, tile->tileset->_cache, tile); + GC_CHECK_ERROR(ctx); + } + } + +cleanup: + return; +} + +void mapcache_tileset_tile_get_with_subdimensions(mapcache_context *ctx, mapcache_tile *tile) { + int i,ret; + assert(tile->dimensions); + if(tile->tileset->store_dimension_assemblies) { + for(i=0;idimensions->nelts;i++) { + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + dim->cached_value = dim->requested_value; + } + ret = mapcache_tileset_tile_get_readonly(ctx,tile); + GC_CHECK_ERROR(ctx); + if(ret == MAPCACHE_SUCCESS) { + /* update the tile expiration time */ + if(tile->tileset->auto_expire && tile->mtime) { + apr_time_t now = apr_time_now(); + apr_time_t expire_time = tile->mtime + apr_time_from_sec(tile->tileset->auto_expire); + tile->expires = apr_time_sec(expire_time-now); + } + return; + } + for(i=0;idimensions->nelts;i++) { + /* unset the cached dimension we setup earlier on */ + mapcache_requested_dimension *dim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + dim->cached_value = NULL; + } + } + return mapcache_tileset_tile_set_get_with_subdimensions(ctx,tile); + +} + /** * \brief return the image data for a given tile * this call uses a global (interprocess+interthread) mutex if the tile was not found @@ -761,16 +1004,12 @@ * - release mutex * */ -void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile) + +static void mapcache_tileset_tile_get_without_subdimensions(mapcache_context *ctx, mapcache_tile *tile, int read_only) { int ret; mapcache_metatile *mt=NULL; - if(tile->grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && - tile->z > tile->grid_link->max_cached_zoom) { - mapcache_tileset_outofzoom_get(ctx, tile); - return; - } - ret = tile->tileset->_cache->tile_get(ctx, tile->tileset->_cache, tile); + ret = mapcache_cache_tile_get(ctx, tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(ret == MAPCACHE_SUCCESS && tile->tileset->auto_expire && tile->mtime && tile->tileset->source && !tile->tileset->read_only) { @@ -787,7 +1026,7 @@ if (ret == MAPCACHE_CACHE_MISS) { /* bail out straight away if the tileset has no source or is read-only */ - if(tile->tileset->read_only || !tile->tileset->source) { + if(read_only) { /* there is no source configured for this tile. not an error, let caller now*/ /* ctx->set_error(ctx,404,"tile not in cache, and no source configured for tileset %s", @@ -803,14 +1042,14 @@ return; } } - - + + if (ret == MAPCACHE_CACHE_MISS || ret == MAPCACHE_CACHE_RELOAD) { - int isLocked; + int isLocked = MAPCACHE_FALSE; void *lock; /* If the tile does not exist or stale, we must take action before re-asking for it */ - if( !tile->tileset->read_only && tile->tileset->source && !ctx->config->non_blocking) { + if( !read_only && !ctx->config->non_blocking) { /* * is the tile already being rendered by another thread ? * the call is protected by the same mutex that sets the lock on the tile, @@ -838,25 +1077,25 @@ /* temporarily clear error state so we don't mess up with error handling in the locker */ void *error; ctx->pop_errors(ctx,&error); - mapcache_unlock_resource(ctx, ctx->config->locker, mapcache_tileset_metatile_resource_key(ctx,mt), lock); + mapcache_unlock_resource(ctx, ctx->config->locker, lock); ctx->push_errors(ctx,error); } else { - mapcache_unlock_resource(ctx, ctx->config->locker, mapcache_tileset_metatile_resource_key(ctx,mt), lock); + mapcache_unlock_resource(ctx, ctx->config->locker, lock); } } } - if (ret == MAPCACHE_CACHE_RELOAD && GC_HAS_ERROR(ctx)) - /* If we tried to reload a stale tile but failed, we know we have already - * fetched it from the cache. We can then ignore errors and just use old tile. + if (ret == MAPCACHE_CACHE_RELOAD && GC_HAS_ERROR(ctx)) + /* If we tried to reload a stale tile but failed, we know we have already + * fetched it from the cache. We can then ignore errors and just use old tile. */ ctx->clear_errors(ctx); else { - /* Else, check for errors and try to fetch the tile from the cache. - */ - GC_CHECK_ERROR(ctx); - ret = tile->tileset->_cache->tile_get(ctx, tile->tileset->_cache, tile); + /* Else, check for errors and try to fetch the tile from the cache. + */ + GC_CHECK_ERROR(ctx); + ret = mapcache_cache_tile_get(ctx, tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(ret != MAPCACHE_SUCCESS) { @@ -878,11 +1117,48 @@ } } +void mapcache_tileset_tile_get(mapcache_context *ctx, mapcache_tile *tile) { + if(tile->grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && + tile->z > tile->grid_link->max_cached_zoom) { + mapcache_tileset_outofzoom_get(ctx, tile); + return; + } + if(tile->dimensions) { + if(tile->tileset->dimension_assembly_type != MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + return mapcache_tileset_tile_get_with_subdimensions(ctx,tile); + } else { + int i; + mapcache_requested_dimension *rdim; + mapcache_extent extent; + + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + for(i=0; idimensions->nelts; i++) { + apr_array_header_t *rdim_vals; + rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim_vals = rdim->dimension->get_entries_for_value(ctx,rdim->dimension,rdim->requested_value, tile->tileset, NULL, tile->grid_link->grid); + GC_CHECK_ERROR(ctx); + if(rdim_vals->nelts > 1) { + ctx->set_error(ctx,500,"dimension (%s) for tileset (%s) returned invalid number (%d) of subdimensions (1 expected)", + rdim->dimension->name, tile->tileset->name, rdim_vals->nelts); + return; + } + if(rdim_vals->nelts == 0) { + ctx->set_error(ctx,404,"dimension (%s) for tileset (%s) returned no subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return; + } + rdim->cached_value = APR_ARRAY_IDX(rdim_vals,0,char*); + } + } + } + return mapcache_tileset_tile_get_without_subdimensions(ctx,tile, (tile->tileset->read_only||!tile->tileset->source)?1:0); + +} + void mapcache_tileset_tile_delete(mapcache_context *ctx, mapcache_tile *tile, int whole_metatile) { int i; /*delete the tile itself*/ - tile->tileset->_cache->tile_delete(ctx,tile->tileset->_cache, tile); + mapcache_cache_tile_delete(ctx,tile->tileset->_cache, tile); GC_CHECK_ERROR(ctx); if(whole_metatile) { @@ -891,7 +1167,7 @@ mapcache_tile *subtile = &mt->tiles[i]; /* skip deleting the actual tile */ if(subtile->x == tile->x && subtile->y == tile->y) continue; - subtile->tileset->_cache->tile_delete(ctx,subtile->tileset->_cache,subtile); + mapcache_cache_tile_delete(ctx,subtile->tileset->_cache,subtile); /* silently pass failure if the tile was not found */ if(ctx->get_error(ctx) == 404) { ctx->clear_errors(ctx); diff -Nru mapcache-1.4.0/lib/util.c mapcache-1.6.1/lib/util.c --- mapcache-1.4.0/lib/util.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/lib/util.c 2017-09-29 21:31:13.000000000 +0000 @@ -33,6 +33,7 @@ #include #include #include +#include #ifndef _WIN32 #include @@ -41,6 +42,14 @@ #ifndef M_PI #define M_PI 3.14159265358979323846264338327 #endif + +#ifdef _WIN32 +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long int uint64_t; +#endif + const double mapcache_meters_per_unit[MAPCACHE_UNITS_COUNT] = {1.0,6378137.0 * 2.0 * M_PI / 360,0.3048}; @@ -65,11 +74,15 @@ for (i = 0, j = 0; i < input_length;) { - uint32_t octet_a = i < input_length ? (unsigned char)data[i++] : 0; - uint32_t octet_b = i < input_length ? (unsigned char)data[i++] : 0; - uint32_t octet_c = i < input_length ? (unsigned char)data[i++] : 0; + uint32_t octet_a; + uint32_t octet_b; + uint32_t octet_c; + uint32_t triple; + octet_a = i < input_length ? (unsigned char)data[i++] : 0; + octet_b = i < input_length ? (unsigned char)data[i++] : 0; + octet_c = i < input_length ? (unsigned char)data[i++] : 0; - uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; + triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; encoded_data[j++] = encoding_table[(triple >> 3 * 6) & 0x3F]; encoded_data[j++] = encoding_table[(triple >> 2 * 6) & 0x3F]; @@ -79,7 +92,7 @@ for (i = 0; i < mod_table[input_length % 3]; i++) encoded_data[output_length - 2 - i] = '='; - + encoded_data[output_length-1]=0; return encoded_data; @@ -172,6 +185,53 @@ return pstr; } +char* mapcache_util_str_xml_escape(apr_pool_t *pool, const char *str, + mapcache_util_xml_section_type xml_section_type) +{ + int outpos = 0; + char* outstr = apr_pcalloc(pool, 6 * strlen(str) + 1); + for( ; *str != '\0'; str ++ ) { + if( xml_section_type == MAPCACHE_UTIL_XML_SECTION_COMMENT ) { + if( *str == '-' ) { + memcpy(outstr + outpos, "-", 5); + outpos += 5; + } + else { + outstr[outpos] = *str; + outpos ++; + } + } + else { + if( *str == '&' ) { + memcpy(outstr + outpos, "&", 5); + outpos += 5; + } + else if( *str == '<' ) { + memcpy(outstr + outpos, "<", 4); + outpos += 4; + } + else if( *str == '>' ) { + memcpy(outstr + outpos, ">", 4); + outpos += 4; + } + else if( *str == '"' ) { + memcpy(outstr + outpos, """, 6); + outpos += 6; + } + else if( *str == '\'' ) { + /* See https://github.com/mapserver/mapserver/issues/1040 */ + memcpy(outstr + outpos, "'", 5); + outpos += 5; + } + else { + outstr[outpos] = *str; + outpos ++; + } + } + } + return outstr; +} + #if APR_MAJOR_VERSION < 1 || (APR_MAJOR_VERSION < 2 && APR_MINOR_VERSION < 3) APR_DECLARE(apr_table_t *) apr_table_clone(apr_pool_t *p, const apr_table_t *t) { @@ -261,10 +321,11 @@ void _mapcache_context_push_errors(mapcache_context *ctx, void *error) { struct _error_log *e = (struct _error_log*)error; - ctx->_errcode = e->_errcode; + if(e->_errcode) + ctx->_errcode = e->_errcode; if(e->_errmsg) { if(ctx->_errmsg) { - ctx->_errmsg = apr_psprintf(ctx->pool,"%s\n%s",ctx->_errmsg,e->_errmsg); + ctx->_errmsg = apr_psprintf(ctx->pool,"%s\n%s",e->_errmsg,ctx->_errmsg); } else { ctx->_errmsg = e->_errmsg; } @@ -310,7 +371,6 @@ dst->set_exception = src->set_exception; dst->service = src->service; dst->exceptions = src->exceptions; - dst->threadlock = src->threadlock; dst->supports_redirects = src->supports_redirects; dst->pop_errors = src->pop_errors; dst->push_errors = src->push_errors; @@ -322,21 +382,20 @@ { char *key = apr_pstrdup(ctx->pool,""); if(tile->dimensions) { - const apr_array_header_t *elts = apr_table_elts(tile->dimensions); - int i = elts->nelts; + int i = tile->dimensions->nelts; if(i>1) { while(i--) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,i,apr_table_entry_t)); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); if(i) { - key = apr_pstrcat(ctx->pool,key,entry->val,(sanitized_chars?sanitize_to:"#"),NULL); + key = apr_pstrcat(ctx->pool,key,entry->cached_value,(sanitized_chars?sanitize_to:"#"),NULL); } else { - key = apr_pstrcat(ctx->pool,key,entry->val,NULL); + key = apr_pstrcat(ctx->pool,key,entry->cached_value,NULL); } } return key; } else if(i) { - apr_table_entry_t *entry = &(APR_ARRAY_IDX(elts,0,apr_table_entry_t)); - key = apr_pstrdup(ctx->pool,entry->val); + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,0,mapcache_requested_dimension*); + key = apr_pstrdup(ctx->pool,entry->cached_value); } if(sanitized_chars) key = mapcache_util_str_sanitize(ctx->pool,key,sanitized_chars,*sanitize_to); @@ -344,19 +403,109 @@ return key; } +void mapcache_util_quadkey_decode(mapcache_context *ctx, const char *quadkey, int *x, int *y, int *z) { + int i; + if(!quadkey || !*quadkey) { + *z = *x = *y = 0; + return; + } + *z = strlen(quadkey); + *x = *y = 0; + for (i = *z; i; i--) { + int mask = 1 << (i - 1); + switch (quadkey[*z - i]) { + case '0': + break; + case '1': + *x |= mask; + break; + case '2': + *y |= mask; + break; + case '3': + *x |= mask; + *y |= mask; + break; + default: + ctx->set_error(ctx, 400, "Invalid Quadkey sequence"); + return; + } + } +} + +char* mapcache_util_quadkey_encode(mapcache_context *ctx, int x, int y, int z) { + int i; + char *key = apr_pcalloc(ctx->pool, z+1); + memset(key,'0',z); + for (i = z; i > 0; i--) { + int mask = 1 << (i - 1); + if ((x & mask) != 0) + { + key[z - i]++; + } + if ((y & mask) != 0) + { + key[z - i] += 2; + } + } + return key; +} + char* mapcache_util_get_tile_key(mapcache_context *ctx, mapcache_tile *tile, char *template, char* sanitized_chars, char *sanitize_to) { char *path; if(template) { - path = mapcache_util_str_replace(ctx->pool, template, "{x}", - apr_psprintf(ctx->pool, "%d", tile->x)); - path = mapcache_util_str_replace(ctx->pool, path, "{y}", - apr_psprintf(ctx->pool, "%d", tile->y)); - path = mapcache_util_str_replace(ctx->pool, path, "{z}", - apr_psprintf(ctx->pool, "%d", tile->z)); - if(strstr(path,"{dim}")) { - path = mapcache_util_str_replace(ctx->pool, path, "{dim}", mapcache_util_get_tile_dimkey(ctx,tile,sanitized_chars,sanitize_to)); + path = apr_pstrdup(ctx->pool, template); + + if(strstr(path,"{x}")) + path = mapcache_util_str_replace(ctx->pool,path, "{x}", + apr_psprintf(ctx->pool,"%d",tile->x)); + else if(strstr(path,"{inv_x}")) + path = mapcache_util_str_replace(ctx->pool,path, "{inv_x}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxx - tile->x - 1)); + if(strstr(path,"{y}")) + path = mapcache_util_str_replace(ctx->pool,path, "{y}", + apr_psprintf(ctx->pool,"%d",tile->y)); + else if(strstr(path,"{inv_y}")) + path = mapcache_util_str_replace(ctx->pool,path, "{inv_y}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->levels[tile->z]->maxy - tile->y - 1)); + if(strstr(path,"{z}")) + path = mapcache_util_str_replace(ctx->pool,path, "{z}", + apr_psprintf(ctx->pool,"%d",tile->z)); + else if(strstr(path,"{inv_z}")) + path = mapcache_util_str_replace(ctx->pool,path, "{inv_z}", + apr_psprintf(ctx->pool,"%d", + tile->grid_link->grid->nlevels - tile->z - 1)); + if(strstr(path,"{quadkey}")) { + char *quadkey = mapcache_util_quadkey_encode(ctx, tile->x, tile->y, tile->z); + path = mapcache_util_str_replace(ctx->pool,path, "{quadkey}", quadkey); + } + + if(tile->dimensions) { + if(strstr(path,"{dim:")) { + char *dimstring=""; + int i = tile->dimensions->nelts; + while(i--) { + char *single_dim; + mapcache_requested_dimension *entry = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + + /* compute value for eventual {dim} replacement */ + dimstring = apr_pstrcat(ctx->pool,dimstring,"#",entry->dimension->name,"#",entry->cached_value,NULL); + + /* check for {dim:name} replacement */ + single_dim = apr_pstrcat(ctx->pool,"{dim:",entry->dimension->name,"}",NULL); + if(strstr(path,single_dim)) { + path = mapcache_util_str_replace(ctx->pool,path, single_dim, entry->cached_value); + } + } + } + if(strstr(path,"{dim}")) { + path = mapcache_util_str_replace(ctx->pool,path, "{dim}", + mapcache_util_get_tile_dimkey(ctx,tile,sanitized_chars,sanitize_to)); + } } if(strstr(path,"{tileset}")) path = mapcache_util_str_replace(ctx->pool, path, "{tileset}", tile->tileset->name); @@ -387,9 +536,41 @@ return path; } +void mapcache_make_parent_dirs(mapcache_context *ctx, char *filename) { + char *hackptr1,*hackptr2=NULL; + apr_status_t ret; + char errmsg[120]; + + /* find the location of the last '/' in the string */ + hackptr1 = filename; + while(*hackptr1) { + if(*hackptr1 == '/') + hackptr2 = hackptr1; + hackptr1++; + } + + if(hackptr2) { + /* terminate string on last '/' */ + *hackptr2 = '\0'; + } -/* vim: ts=2 sts=2 et sw=2 -*/ + ret = apr_dir_make_recursive(filename,APR_OS_DEFAULT,ctx->pool); + + if(hackptr2) { + *hackptr2 = '/'; + } + + + if(APR_SUCCESS != ret) { + /* + * apr_dir_make_recursive sometimes sends back this error, although it should not. + * ignore this one + */ + if(!APR_STATUS_IS_EEXIST(ret)) { + ctx->set_error(ctx, 500, "failed to create directory %s: %s",filename, apr_strerror(ret,errmsg,120)); + } + } +} #if defined(_WIN32) && !defined(__CYGWIN__) @@ -443,3 +624,8 @@ #endif + +/* vim: ts=2 sts=2 et sw=2 +*/ + + diff -Nru mapcache-1.4.0/mapcache.xml.sample mapcache-1.6.1/mapcache.xml.sample --- mapcache-1.4.0/mapcache.xml.sample 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/mapcache.xml.sample 2017-09-29 21:31:13.000000000 +0000 @@ -523,6 +523,31 @@ + + + + + + + + + + /dev/shm/google-mapcache.header + 10 + 3 + + + + + + + + GOOGPGDWFDG345SDFGSD + sdfgsdSDFwedfwefr2345324dfsGdsfgs + 10 + 3 + + + true best diff -Nru mapcache-1.4.0/MIGRATION_GUIDE.txt mapcache-1.6.1/MIGRATION_GUIDE.txt --- mapcache-1.4.0/MIGRATION_GUIDE.txt 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/MIGRATION_GUIDE.txt 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,16 @@ +Migrating from Mapcache 1.4 to 1.6 +=================================== + +* The tileset child has been removed. Time dimensions are now added with + +* val1,val2,val3 should be replaced by + val1val2val3 + +* ... should be replaced by + true.... + +* ^abc$ should be replaced by + ^abc$ + +* ... should be replaced by + stack.... diff -Nru mapcache-1.4.0/nginx/ngx_http_mapcache_module.c mapcache-1.6.1/nginx/ngx_http_mapcache_module.c --- mapcache-1.4.0/nginx/ngx_http_mapcache_module.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/nginx/ngx_http_mapcache_module.c 2017-09-29 21:31:13.000000000 +0000 @@ -63,7 +63,6 @@ mapcache_context *ctx = apr_pcalloc(process_pool, sizeof(mapcache_ngx_context)); ctx->pool = process_pool; ctx->connection_pool = NULL; - ctx->threadlock = NULL; mapcache_context_init(ctx); ctx->log = ngx_mapcache_context_log; ctx->clone = ngx_mapcache_context_clone; @@ -297,6 +296,10 @@ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,ctx->get_error_message(ctx)); return NGX_CONF_ERROR; } + if(mapcache_config_services_enabled(ctx, ctx->config) <= 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, "no mapcache s configured/enabled, no point in continuing."); + return NGX_CONF_ERROR; + } mapcache_connection_pool_create(&ctx->connection_pool,ctx->pool); ctx->config->non_blocking = 1; diff -Nru mapcache-1.4.0/release.sh mapcache-1.6.1/release.sh --- mapcache-1.4.0/release.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/release.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,54 @@ +#!/bin/bash + +ms_version=$1 + +echo "release steps for mapcache version $ms_version (mapcache-$ms_version.tar.gz)" +echo "" + +ms_version_suffix=`echo $ms_version | cut -s -d- -f2` +if [ -z $ms_version_suffix ]; then + ms_version_num=$ms_version +else + ms_version_num=`echo $ms_version | cut -s -d- -f1` +fi + +ms_version_major=`echo $ms_version_num | cut -d. -f1` +ms_version_minor=`echo $ms_version_num | cut -d. -f2` +ms_version_revision=`echo $ms_version_num | cut -d. -f3` + + +tagname=rel-$ms_version_major-$ms_version_minor-$ms_version_revision +if [ ! -z $ms_version_suffix ]; then + tagname=$tagname-$ms_version_suffix +fi + +echo "#" +echo "# make sure:" +echo "# - you are on branch-$ms_version_major-$ms_version_minor" +echo "# - you have edited HISTORY.TXT with changes related to this release" +echo "#" +echo "" + +echo "sed -i '/set (MAPCACHE_VERSION_MAJOR/c\set (MAPCACHE_VERSION_MAJOR $ms_version_major)' CMakeLists.txt" +echo "sed -i '/set (MAPCACHE_VERSION_MINOR/c\set (MAPCACHE_VERSION_MINOR $ms_version_minor)' CMakeLists.txt" +echo "sed -i '/set (MAPCACHE_VERSION_REVISION/c\set (MAPCACHE_VERSION_REVISION $ms_version_revision)' CMakeLists.txt" +if [ ! -z $ms_version_suffix ]; then + echo "sed -i '/set (MAPCACHE_VERSION_SUFFIX/c\set (MAPCACHE_VERSION_SUFFIX \"-$ms_version_suffix\")' CMakeLists.txt" +else + echo "sed -i '/set (MAPCACHE_VERSION_SUFFIX/c\set (MAPCACHE_VERSION_SUFFIX \"\")' CMakeLists.txt" +fi + +echo "git add CMakeLists.txt" +echo "git commit -m \"update for $ms_version release\"" +echo "git tag -a $tagname -m \"Create $ms_version tag\"" +echo "git push origin branch-$ms_version_major-$ms_version_minor --tags" +echo "git archive --format=tar.gz --prefix=mapcache-$ms_version/ $tagname >/tmp/mapcache-$ms_version.tar.gz" +echo "scp /tmp/mapcache-$ms_version.tar.gz download.osgeo.org:/osgeo/download/mapserver" + +echo "" +echo "#" +echo "#optionally update doc site, these commands need tweaking before being ran" +echo "#" + +echo "/path/to/docs/scripts/changelog.sh rel-previous-tag..$tagname >> /path/to/docs/en/development/changelog/changelog-$ms_version_major-$ms_version_minor.txt" + Binary files /tmp/tmpkFMa7Z/i8mYsTwVBp/mapcache-1.4.0/tests/data/world.tif and /tmp/tmpkFMa7Z/YNoG7H7JdS/mapcache-1.6.1/tests/data/world.tif differ diff -Nru mapcache-1.4.0/tests/expected/wms_capabilities.xml mapcache-1.6.1/tests/expected/wms_capabilities.xml --- mapcache-1.4.0/tests/expected/wms_capabilities.xml 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/expected/wms_capabilities.xml 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,75 @@ + + +]> + + + OGC:WMS + no title set, add some in metadata + + + + + + application/vnd.ogc.wms_xml + + + + + + + + + + image/png + image/jpeg + + + + + + + + + + text/plain + application/vnd.ogc.gml + + + + + + + + + + + text/plain + + + + EPSG:3857 + + 156543.03392804099712520838 78271.51696402048401068896 39135.75848201022745342925 19567.87924100512100267224 9783.93962050256050133612 4891.96981025128025066806 2445.98490512564012533403 1222.99245256282006266702 611.49622628141003133351 305.74811314070478829308 152.87405657035250783338 76.43702828517623970583 38.21851414258812695834 19.10925707129405992646 9.55462853564703173959 4.77731426782351586979 2.38865713391175793490 1.19432856695587897633 + 256 + 256 + image/jpeg + global + + + + + no title set, add some in metadata + EPSG:3857 + EPSG:4326 + EPSG:900913 + + global + global + + EPSG:3857 + EPSG:900913 + + + + diff -Nru mapcache-1.4.0/tests/expected/wmts_capabilities.xml mapcache-1.6.1/tests/expected/wmts_capabilities.xml --- mapcache-1.4.0/tests/expected/wmts_capabilities.xml 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/expected/wmts_capabilities.xml 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,247 @@ + + + + OGC WMTS + 1.0.0 + + + + + + + + + + + + + + KVP + + + + + + + + + + + + + KVP + + + + + + + + + + + + + KVP + + + + + + + + + + global + global + + image/jpeg + + GoogleMapsCompatible + + + + + GoogleMapsCompatible + + -20037508.342789 -20037508.342789 + 20037508.342789 20037508.342789 + + urn:ogc:def:crs:EPSG:6.3:3857 + urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible + + 0 + 559082264.02871787548065185547 + -20037508.342789 20037508.342789 + 256 + 256 + 1 + 1 + + + 1 + 279541132.01435887813568115234 + -20037508.342789 20037508.342789 + 256 + 256 + 2 + 2 + + + 2 + 139770566.00717940926551818848 + -20037508.342789 20037508.342789 + 256 + 256 + 4 + 4 + + + 3 + 69885283.00358971953392028809 + -20037508.342789 20037508.342789 + 256 + 256 + 8 + 8 + + + 4 + 34942641.50179485976696014404 + -20037508.342789 20037508.342789 + 256 + 256 + 16 + 16 + + + 5 + 17471320.75089742988348007202 + -20037508.342789 20037508.342789 + 256 + 256 + 32 + 32 + + + 6 + 8735660.37544871494174003601 + -20037508.342789 20037508.342789 + 256 + 256 + 64 + 64 + + + 7 + 4367830.18772435747087001801 + -20037508.342789 20037508.342789 + 256 + 256 + 128 + 128 + + + 8 + 2183915.09386217873543500900 + -20037508.342789 20037508.342789 + 256 + 256 + 256 + 256 + + + 9 + 1091957.54693108866922557354 + -20037508.342789 20037508.342789 + 256 + 256 + 512 + 512 + + + 10 + 545978.77346554468385875225 + -20037508.342789 20037508.342789 + 256 + 256 + 1024 + 1024 + + + 11 + 272989.38673277228372171521 + -20037508.342789 20037508.342789 + 256 + 256 + 2048 + 2048 + + + 12 + 136494.69336638617096468806 + -20037508.342789 20037508.342789 + 256 + 256 + 4096 + 4096 + + + 13 + 68247.34668319307093042880 + -20037508.342789 20037508.342789 + 256 + 256 + 8192 + 8192 + + + 14 + 34123.67334159654274117202 + -20037508.342789 20037508.342789 + 256 + 256 + 16384 + 16384 + + + 15 + 17061.83667079827137058601 + -20037508.342789 20037508.342789 + 256 + 256 + 32768 + 32768 + + + 16 + 8530.91833539913568529300 + -20037508.342789 20037508.342789 + 256 + 256 + 65536 + 65536 + + + 17 + 4265.45916769956784264650 + -20037508.342789 20037508.342789 + 256 + 256 + 131072 + 131072 + + + 18 + 2132.72958384978392132325 + -20037508.342789 20037508.342789 + 256 + 256 + 262144 + 262144 + + + + diff -Nru mapcache-1.4.0/tests/run_tests.sh mapcache-1.6.1/tests/run_tests.sh --- mapcache-1.4.0/tests/run_tests.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/run_tests.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,49 @@ +#!/bin/sh + +# Project: MapCache +# Purpose: MapCache tests +# Author: Even Rouault +# +#***************************************************************************** +# Copyright (c) 2017 Regents of the University of Minnesota. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +#****************************************************************************/ + +set -e + +MAPCACHE_CONF=/tmp/mc/mapcache.xml + +sudo rm -rf /tmp/mc/global +mapcache_seed -c /tmp/mc/mapcache.xml -t global --force -z 0,1 +gdalinfo -checksum /tmp/mc/global/GoogleMapsCompatible/00/000/000/000/000/000/000.jpg | grep Checksum=20574 >/dev/null || (echo "Did not get expected checksum"; gdalinfo -checksum /tmp/mc/global/GoogleMapsCompatible/00/000/000/000/000/000/000.jpg; /bin/false) +sudo rm -rf /tmp/mc/global + +curl -s "http://localhost/mapcache/?SERVICE=WMS&REQUEST=GetCapabilities" | xmllint --format - > /tmp/wms_capabilities.xml +diff -u /tmp/wms_capabilities.xml expected + +curl -s "http://localhost/mapcache/wmts?SERVICE=WMTS&REQUEST=GetCapabilities" | xmllint --format - > /tmp/wmts_capabilities.xml +diff -u /tmp/wmts_capabilities.xml expected + +curl -s "http://localhost/mapcache/wmts/1.0.0/global/default/GoogleMapsCompatible/0/0/0.jpg" > /tmp/0.jpg +gdalinfo -checksum /tmp/0.jpg | grep Checksum=20574 >/dev/null || (echo "Did not get expected checksum"; gdalinfo -checksum /tmp/0.jpg; /bin/false) + +curl -s "http://localhost/mapcache/wmts/1.0.0/global/default/GoogleMapsCompatible/0/0/0.jpg" > /tmp/0_bis.jpg +diff /tmp/0.jpg /tmp/0_bis.jpg + diff -Nru mapcache-1.4.0/tests/travis_setup.sh mapcache-1.6.1/tests/travis_setup.sh --- mapcache-1.4.0/tests/travis_setup.sh 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/tests/travis_setup.sh 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,65 @@ +#!/bin/sh + +# Project: MapCache +# Purpose: MapCache tests +# Author: Even Rouault +# +#***************************************************************************** +# Copyright (c) 2017 Regents of the University of Minnesota. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies of this Software or works derived from this Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +#****************************************************************************/ + +set -e + +mkdir /tmp/mc +sudo chmod -R a+rw /tmp/mc + +MAPCACHE_CONF=/tmp/mc/mapcache.xml +echo '' >> $MAPCACHE_CONF +echo '' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' /tmp/mc/world.tif' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' /tmp/mc' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' disk' >> $MAPCACHE_CONF +echo ' global-tif' >> $MAPCACHE_CONF +echo ' GoogleMapsCompatible' >> $MAPCACHE_CONF +echo ' JPEG' >> $MAPCACHE_CONF +echo ' 1 1' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' ' >> $MAPCACHE_CONF +echo ' debug' >> $MAPCACHE_CONF +echo '' >> $MAPCACHE_CONF + +cp data/world.tif /tmp/mc + +sudo su -c "echo 'LoadModule mapcache_module /usr/lib/apache2/modules/mod_mapcache.so' >> /etc/apache2/apache2.conf" +sudo su -c "echo '' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' ' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' Require all granted' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' ' >> /etc/apache2/apache2.conf" +sudo su -c "echo ' MapCacheAlias /mapcache \"/tmp/mc/mapcache.xml\"' >> /etc/apache2/apache2.conf" +sudo su -c "echo '' >> /etc/apache2/apache2.conf" + +sudo service apache2 restart diff -Nru mapcache-1.4.0/.travis.yml mapcache-1.6.1/.travis.yml --- mapcache-1.4.0/.travis.yml 1970-01-01 00:00:00.000000000 +0000 +++ mapcache-1.6.1/.travis.yml 2017-09-29 21:31:13.000000000 +0000 @@ -0,0 +1,56 @@ +matrix: + fast_finish: true + include: + - os: linux + dist: trusty + language: c + sudo: required + env: + - DISTRO=trusty + - BUILD_TYPE=maximum + + - os: linux + dist: trusty + language: c + sudo: required + env: + - DISTRO=trusty + - BUILD_TYPE=minimum + + - os: linux + language: c + sudo: required + env: + - DISTRO=precise + - BUILD_TYPE=maximum + +language: c + +before_install: + - sudo mv /etc/apt/sources.list.d/pgdg* /tmp + - sudo apt-get purge -y libgdal* libgeos* libspatialite* + - sudo add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable + - sudo apt-get update + - sudo apt-get install cmake libspatialite-dev libfcgi-dev libproj-dev libgeos-dev libgdal-dev libtiff-dev libgeotiff-dev apache2-dev libpcre3-dev libsqlite3-dev libdb-dev +# For testing + - sudo apt-get install libxml2-utils apache2 gdal-bin + +script: + - mkdir build + - cd build + - if test "$BUILD_TYPE" = "maximum"; then cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DWITH_TIFF=ON -DWITH_GEOTIFF=ON -DWITH_TIFF_WRITE_SUPPORT=ON -DWITH_PCRE=ON -DWITH_SQLITE=ON -DWITH_BERKELEY_DB=ON; else cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DWITH_TIFF=OFF -DWITH_GEOTIFF=OFF -DWITH_TIFF_WRITE_SUPPORT=OFF -DWITH_PCRE=OFF -DWITH_SQLITE=OFF -DWITH_BERKELEY_DB=OFF -DWITH_GDAL=OFF -DWITH_GEOS=OFF -DWITH_FCGI=OFF -DWITH_CGI=OFF -DWITH_APACHE=OFF -DWITH_OGR=OFF -DWITH_MAPSERVER=OFF; fi + - make -j3 + - sudo make install +# Only test with Apache 2.4 + - if test "$DISTRO" = "trusty" -a "$BUILD_TYPE" = "maximum"; then cd ../tests; sh ./travis_setup.sh; sh ./run_tests.sh; fi + + +#notifications: +# email: +# recipients: +# - thomas.bonfort@gmail.com +# irc: +# channels: +# - "irc.freenode.org#mapserver" +# use_notice: true + diff -Nru mapcache-1.4.0/util/mapcache_seed.c mapcache-1.6.1/util/mapcache_seed.c --- mapcache-1.4.0/util/mapcache_seed.c 2015-07-28 13:03:13.000000000 +0000 +++ mapcache-1.6.1/util/mapcache_seed.c 2017-09-29 21:31:13.000000000 +0000 @@ -37,6 +37,7 @@ #include #ifndef _WIN32 #include +#include #define USE_FORK #include #endif @@ -70,7 +71,7 @@ mapcache_tileset *tileset_transfer; mapcache_cfg *cfg; mapcache_context ctx; -apr_table_t *dimensions=NULL; +apr_array_header_t *dimensions=NULL; int minzoom=-1; int maxzoom=-1; mapcache_grid_link *grid_link; @@ -82,7 +83,9 @@ int sig_int_received = 0; int error_detected = 0; double percent_failed_allowed = 1.0; -int n_metatiles_tot=0; +int n_metatiles_tot = 0; +int n_nodata_tot = 0; +int rate_limit = 0; FILE *failed_log = NULL, *retry_log = NULL; #define FAIL_BACKLOG_COUNT 1000 @@ -94,7 +97,8 @@ MAPCACHE_CMD_STOP, MAPCACHE_CMD_DELETE, MAPCACHE_CMD_SKIP, - MAPCACHE_CMD_TRANSFER + MAPCACHE_CMD_TRANSFER, + MAPCACHE_CMD_STOP_RECURSION } cmd; typedef enum { @@ -122,6 +126,7 @@ struct seed_status { s_status status; int x,y,z; + int nodata; char *msg; }; @@ -136,6 +141,9 @@ int push_queue(struct seed_cmd cmd) { + struct seed_cmd *pcmd; + int retries=0; + int ret; #ifdef USE_FORK if(nprocesses > 1) { struct msg_cmd mcmd; @@ -148,14 +156,23 @@ return APR_SUCCESS; } #endif - struct seed_cmd *pcmd = calloc(1,sizeof(struct seed_cmd)); + pcmd = calloc(1,sizeof(struct seed_cmd)); *pcmd = cmd; - return apr_queue_push(work_queue,pcmd); + ret = apr_queue_push(work_queue,pcmd); + while (ret == APR_EINTR && retries < 10) { + retries++; + ret = apr_queue_push(work_queue,pcmd); + } + if(ret == APR_EINTR) { + printf("failed to push tile %d %d %d after 10 retries\n",cmd.z,cmd.y,cmd.x); + return APR_EGENERAL; + } + return ret; } int pop_queue(struct seed_cmd *cmd) { - int ret; + int ret,retries=0; struct seed_cmd *pcmd; #ifdef USE_FORK @@ -169,8 +186,12 @@ return APR_SUCCESS; } #endif - + ret = apr_queue_pop(work_queue, (void**)&pcmd); + while(ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_pop(work_queue, (void**)&pcmd); + } if(ret == APR_SUCCESS) { *cmd = *pcmd; free(pcmd); @@ -180,7 +201,7 @@ int trypop_queue(struct seed_cmd *cmd) { - int ret; + int ret,retries=0; struct seed_cmd *pcmd; #ifdef USE_FORK @@ -198,6 +219,10 @@ } #endif ret = apr_queue_trypop(work_queue,(void**)&pcmd); + while(ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_trypop(work_queue,(void**)&pcmd); + } if(ret == APR_SUCCESS) { *cmd = *pcmd; free(pcmd); @@ -205,6 +230,9 @@ return ret; } +#define SEEDER_OPT_THREAD_DELAY 256 +#define SEEDER_OPT_RATE_LIMIT 257 + static const apr_getopt_option_t seed_options[] = { /* long-option, short-option, has-arg flag, description */ { "config", 'c', TRUE, "configuration file (/path/to/mapcache.xml)"}, @@ -217,7 +245,7 @@ { "force", 'f', FALSE, "force tile recreation even if it already exists" }, { "grid", 'g', TRUE, "grid to seed" }, { "help", 'h', FALSE, "show help" }, - { "iteration-mode", 'i', TRUE, "either \"drill-down\" or \"level-by-level\". Default is to use drill-down for g, WGS84 and GoogleMapsCompatible grids, and level-by-level for others. Use this flag to override." }, + { "iteration-mode", 'i', TRUE, "either \"drill-down\" or \"scanline\". Default is to use drill-down for g, WGS84 and GoogleMapsCompatible grids, and scanline for others. Use this flag to override." }, #ifdef USE_CLIPPERS { "ogr-layer", 'l', TRUE, "layer inside datasource"}, #endif @@ -240,14 +268,19 @@ #endif { "transfer", 'x', TRUE, "tileset to transfer" }, { "zoom", 'z', TRUE, "min and max zoomlevels to seed, separated by a comma. eg 0,6" }, + { "rate-limit", SEEDER_OPT_RATE_LIMIT, TRUE, "maximum number of tiles/second to seed"}, + { "thread-delay", SEEDER_OPT_THREAD_DELAY, TRUE, "delay in seconds between rendering thread creation (ramp up)"}, { NULL, 0, 0, NULL } }; void handle_sig_int(int signal) { +#define signal_msg "SIGINT received, waiting for threads to finish\npress ctrl-C again to force terminate\n" if(!sig_int_received) { - fprintf(stderr,"SIGINT received, waiting for threads to finish\n"); - fprintf(stderr,"press ctrl-C again to force terminate, you might end up with locked tiles\n"); + int err = write(2,signal_msg,strlen(signal_msg)); + if(err) { + //nothing we can really do here + } sig_int_received = 1; } else { exit(signal); @@ -256,12 +289,12 @@ void seed_log(mapcache_context *ctx, mapcache_log_level level, char *msg, ...) { - if(verbose) { + if(level >= MAPCACHE_WARN || verbose) { va_list args; va_start(args,msg); vfprintf(stderr,msg,args); va_end(args); - printf("\n"); + fprintf(stderr,"\n"); } } @@ -272,7 +305,7 @@ va_start(args,msg); vfprintf(stderr,msg,args); va_end(args); - printf("\n"); + fprintf(stderr,"\n"); } #ifdef USE_CLIPPERS @@ -280,6 +313,10 @@ { mapcache_metatile *mt = mapcache_tileset_metatile_get(ctx,tile); GEOSCoordSequence *mtbboxls = GEOSCoordSeq_create(5,2); + GEOSGeometry *mtbbox = GEOSGeom_createLinearRing(mtbboxls); + GEOSGeometry *mtbboxg = GEOSGeom_createPolygon(mtbbox,NULL,0); + int i; + int intersects = 0; GEOSCoordSeq_setX(mtbboxls,0,mt->map.extent.minx); GEOSCoordSeq_setY(mtbboxls,0,mt->map.extent.miny); GEOSCoordSeq_setX(mtbboxls,1,mt->map.extent.maxx); @@ -290,10 +327,6 @@ GEOSCoordSeq_setY(mtbboxls,3,mt->map.extent.maxy); GEOSCoordSeq_setX(mtbboxls,4,mt->map.extent.minx); GEOSCoordSeq_setY(mtbboxls,4,mt->map.extent.miny); - GEOSGeometry *mtbbox = GEOSGeom_createLinearRing(mtbboxls); - GEOSGeometry *mtbboxg = GEOSGeom_createPolygon(mtbbox,NULL,0); - int i; - int intersects = 0; for(i=0; i_cache->tile_exists(ctx,tileset->_cache,tile); + int tile_exists; + +#ifdef USE_CLIPPERS + /* check we are in the requested features before checking the tile */ + if(nClippers > 0 && ogr_features_intersect_tile(ctx,tile) == 0) + return MAPCACHE_CMD_STOP_RECURSION; +#endif + + if(mode != MAPCACHE_CMD_TRANSFER && force) { + if(mode == MAPCACHE_CMD_DELETE) { + tile_exists = 1; + } else { + tile_exists = 0; + } + } else { + int i; + if(tile->tileset->dimension_assembly_type != MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim->cached_value = rdim->requested_value; + } + } else { + if(tile->dimensions) { + mapcache_extent extent; + mapcache_grid_get_tile_extent(ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + for(i=0; idimensions->nelts; i++) { + apr_array_header_t *rdim_vals; + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim_vals = rdim->dimension->get_entries_for_value(ctx,rdim->dimension,rdim->requested_value, tile->tileset, NULL, tile->grid_link->grid); + if(GC_HAS_ERROR(ctx)) { + return MAPCACHE_CMD_SKIP; + } + if(rdim_vals->nelts > 1) { + ctx->set_error(ctx,500,"dimension (%s) for tileset (%s) returned invalid number of subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return MAPCACHE_CMD_SKIP; + } + if(rdim_vals->nelts == 0) { + ctx->set_error(ctx,404,"dimension (%s) for tileset (%s) returned no subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return MAPCACHE_CMD_SKIP; + } + rdim->cached_value = APR_ARRAY_IDX(rdim_vals,0,char*); + } + } + } + tile_exists = mapcache_cache_tile_exists(ctx,tileset->_cache,tile); + } /* if the tile exists and a time limit was specified, check the tile modification date */ if(tile_exists) { if(age_limit) { - if(tileset->_cache->tile_get(ctx,tileset->_cache, tile) == MAPCACHE_SUCCESS) { + if(mapcache_cache_tile_get(ctx,tileset->_cache, tile) == MAPCACHE_SUCCESS) { if(tile->mtime && tile->mtime 0) { - intersects = ogr_features_intersect_tile(ctx,tile); - } -#endif - if(intersects != 0) { - /* the tile intersects the ogr features, or there was no clipping asked for: seed it */ - if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { - mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); - /* if we are in mode transfer, delete it from the dst tileset */ - if (mode == MAPCACHE_CMD_TRANSFER) { - tile->tileset = tileset_transfer; - if (tile->tileset->_cache->tile_exists(ctx,tile->tileset->_cache, tile)) { - mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); - } - tile->tileset = tileset; + if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { + mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); + /* if we are in mode transfer, delete it from the dst tileset */ + if (mode == MAPCACHE_CMD_TRANSFER) { + tile->tileset = tileset_transfer; + if (mapcache_cache_tile_exists(ctx,tile->tileset->_cache, tile)) { + mapcache_tileset_tile_delete(ctx,tile,MAPCACHE_TRUE); } - action = mode; - } else { //if(action == MAPCACHE_CMD_DELETE) - action = MAPCACHE_CMD_DELETE; + tile->tileset = tileset; } - } else { - /* the tile does not intersect the ogr features, and already exists, do nothing */ - action = MAPCACHE_CMD_SKIP; + action = mode; + } else { //if(action == MAPCACHE_CMD_DELETE) + action = MAPCACHE_CMD_DELETE; } } } else { @@ -358,7 +423,7 @@ /* the tile exists in the source tileset, check if the tile exists in the destination cache */ tile->tileset = tileset_transfer; - if (tile->tileset->_cache->tile_exists(ctx,tile->tileset->_cache, tile)) { + if (!force && mapcache_cache_tile_exists(ctx,tile->tileset->_cache, tile)) { action = MAPCACHE_CMD_SKIP; } else { action = MAPCACHE_CMD_TRANSFER; @@ -371,21 +436,8 @@ } } else { // the tile does not exist - if(mode == MAPCACHE_CMD_SEED || mode == MAPCACHE_CMD_TRANSFER) { -#ifdef USE_CLIPPERS - /* check we are in the requested features before deleting the tile */ - if(nClippers > 0) { - if(ogr_features_intersect_tile(ctx,tile)) { - action = mode; - } else { - action = MAPCACHE_CMD_SKIP; - } - } else { - action = mode; - } -#else + if (mode == MAPCACHE_CMD_SEED) { action = mode; -#endif } else { action = MAPCACHE_CMD_SKIP; } @@ -394,6 +446,27 @@ return action; } +double rate_limit_last_time = 0; +double rate_limit_delay = 0.0; + +void rate_limit_sleep() { + if(rate_limit > 0) { + struct mctimeval now; + double now_time; + mapcache_gettimeofday(&now,NULL); + now_time = now.tv_sec + now.tv_usec / 1000000.0; + + if((now_time - rate_limit_last_time) < rate_limit_delay) { + apr_sleep((int)((rate_limit_delay - (now_time - rate_limit_last_time)) * 1000000)); + /*last_time = now_time + delay - (now_time - last_time); //now plus the time we slept */ + rate_limit_last_time = rate_limit_delay + rate_limit_last_time; + } else { + rate_limit_last_time = now_time; + } + + } +} + void cmd_recurse(mapcache_context *cmd_ctx, mapcache_tile *tile) { cmd action; @@ -420,8 +493,13 @@ cmd.y = tile->y; cmd.z = tile->z; cmd.command = action; + if(rate_limit > 0) + rate_limit_sleep(); push_queue(cmd); } + + if(action == MAPCACHE_CMD_STOP_RECURSION) + return; //recurse into our 4 child metatiles @@ -440,9 +518,9 @@ */ - mapcache_grid_get_extent(cmd_ctx, grid_link->grid, + mapcache_grid_get_tile_extent(cmd_ctx, grid_link->grid, curx, cury, curz, &bboxbl); - mapcache_grid_get_extent(cmd_ctx, grid_link->grid, + mapcache_grid_get_tile_extent(cmd_ctx, grid_link->grid, curx+tileset->metasize_x-1, cury+tileset->metasize_y-1, curz, &bboxtr); epsilon = (bboxbl.maxx-bboxbl.minx)*0.01; mapcache_grid_get_xy(cmd_ctx,grid_link->grid, @@ -474,7 +552,7 @@ tile->z = curz; } -void cmd_worker() +void feed_worker() { int n; mapcache_tile *tile; @@ -486,7 +564,11 @@ if(nprocesses >= 1) nworkers = nprocesses; apr_pool_create(&cmd_ctx.pool,ctx.pool); tile = mapcache_tileset_tile_create(ctx.pool, tileset, grid_link); - tile->dimensions = dimensions; + tile->dimensions = mapcache_requested_dimensions_clone(ctx.pool,dimensions); + if(rate_limit > 0) { + /* compute time between seed commands accounting for max rate-limit and current metasize */ + rate_limit_delay = (tileset->metasize_x * tileset->metasize_y) / (double)rate_limit; + } if(iteration_mode == MAPCACHE_ITERATION_DEPTH_FIRST) { do { tile->x = x; @@ -534,6 +616,8 @@ cmd.y = y; cmd.z = z; cmd.command = action; + if(rate_limit > 0) + rate_limit_sleep(); push_queue(cmd); } @@ -571,7 +655,9 @@ apr_pool_create(&seed_ctx.pool,ctx.pool); apr_pool_create(&tpool,ctx.pool); tile = mapcache_tileset_tile_create(tpool, tileset, grid_link); - tile->dimensions = dimensions; + if(dimensions) { + tile->dimensions = mapcache_requested_dimensions_clone(tpool,dimensions); + } while(1) { struct seed_cmd cmd; apr_status_t ret; @@ -582,42 +668,63 @@ tile->x = cmd.x; tile->y = cmd.y; tile->z = cmd.z; + tile->nodata = 0; + tile->encoded_data = NULL; + tile->raw_image = NULL; + if(tile->dimensions) { + int i; + if(tileset->dimension_assembly_type == MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + mapcache_extent extent; + mapcache_grid_get_tile_extent(&seed_ctx,tile->grid_link->grid,tile->x,tile->y,tile->z,&extent); + for(i=0; idimensions->nelts; i++) { + apr_array_header_t *rdim_vals; + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim_vals = rdim->dimension->get_entries_for_value(&seed_ctx,rdim->dimension,rdim->requested_value, tile->tileset, NULL, tile->grid_link->grid); + GC_CHECK_ERROR(&seed_ctx); + if(rdim_vals->nelts > 1) { + seed_ctx.set_error(&seed_ctx,500,"dimension (%s) for tileset (%s) returned invalid number of subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return; + } + if(rdim_vals->nelts == 0) { + seed_ctx.set_error(&seed_ctx,404,"dimension (%s) for tileset (%s) returned no subdimensions (1 expected)",rdim->dimension->name, tile->tileset->name); + return; + } + rdim->cached_value = APR_ARRAY_IDX(rdim_vals,0,char*); + } + } else { + for(i=0; idimensions->nelts; i++) { + mapcache_requested_dimension *rdim = APR_ARRAY_IDX(tile->dimensions,i,mapcache_requested_dimension*); + rdim->cached_value = NULL; + } + } + } if(cmd.command == MAPCACHE_CMD_SEED) { - /* aquire a lock on the metatile ?*/ - mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); - void *lock; - int isLocked = mapcache_lock_or_wait_for_resource(&seed_ctx, seed_ctx.config->locker, mapcache_tileset_metatile_resource_key(&seed_ctx,mt), &lock); - if(isLocked == MAPCACHE_TRUE) { + if(!tile->dimensions || tileset->dimension_assembly_type == MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); /* this will query the source to create the tiles, and save them to the cache */ mapcache_tileset_render_metatile(&seed_ctx, mt); - if(GC_HAS_ERROR(&seed_ctx)) { - /* temporarily clear error state so we don't mess up with error handling in the locker */ - void *error; - seed_ctx.pop_errors(&seed_ctx,&error); - mapcache_unlock_resource(&seed_ctx, seed_ctx.config->locker, mapcache_tileset_metatile_resource_key(&seed_ctx,mt), lock); - seed_ctx.push_errors(&seed_ctx,error); - } else { - mapcache_unlock_resource(&seed_ctx, seed_ctx.config->locker, mapcache_tileset_metatile_resource_key(&seed_ctx,mt), lock); - } + } else { + mapcache_tileset_tile_set_get_with_subdimensions(&seed_ctx,tile); } } else if (cmd.command == MAPCACHE_CMD_TRANSFER) { - int i; - mapcache_metatile *mt = mapcache_tileset_metatile_get(&seed_ctx, tile); - for (i = 0; i < mt->ntiles; i++) { - mapcache_tile *subtile = &mt->tiles[i]; - mapcache_tileset_tile_get(&seed_ctx, subtile); - subtile->tileset = tileset_transfer; - subtile->tileset->_cache->tile_set(&seed_ctx, subtile->tileset->_cache, subtile); + mapcache_tileset_tile_get(&seed_ctx, tile); + if(!tile->nodata && !GC_HAS_ERROR(&seed_ctx)) { + mapcache_tileset *tmp_tileset = tile->tileset; + tile->tileset = tileset_transfer; + mapcache_cache_tile_set(&seed_ctx, tile->tileset->_cache, tile); + tile->tileset = tmp_tileset; } } else { //CMD_DELETE mapcache_tileset_tile_delete(&seed_ctx,tile,MAPCACHE_TRUE); } - + { struct seed_status *st = calloc(1,sizeof(struct seed_status)); + int retries=0; st->x=tile->x; st->y=tile->y; st->z=tile->z; + st->nodata = tile->nodata; if(seed_ctx.get_error(&seed_ctx)) { st->status = MAPCACHE_STATUS_FAIL; st->msg = strdup(seed_ctx.get_error_message(&seed_ctx)); @@ -626,6 +733,14 @@ st->status = MAPCACHE_STATUS_OK; } ret = apr_queue_push(log_queue,(void*)st); + while( ret == APR_EINTR && retries < 10) { + retries++; + ret = apr_queue_push(log_queue,(void*)st); + } + if( ret == APR_EINTR) { + printf("FATAL ERROR: unable to log progress after 10 retries, aborting\n"); + break; + } if(ret != APR_SUCCESS) { printf("FATAL ERROR: unable to log progress\n"); @@ -641,25 +756,47 @@ return 0; } #endif + static void* APR_THREAD_FUNC seed_thread(apr_thread_t *thread, void *data) { seed_worker(); return NULL; } +static void* APR_THREAD_FUNC feed_thread_fn(apr_thread_t *thread, void *data) { + //start the thread that will populate the queue. + feed_worker(); + return NULL; +} + static void* APR_THREAD_FUNC log_thread_fn(apr_thread_t *thread, void *data) { + size_t cur; + double last_time; + double now_time; + int i; + int nfailed; + int ntotal; + double pct; char failed[FAIL_BACKLOG_COUNT]; memset(failed,-1,FAIL_BACKLOG_COUNT); - size_t cur=0; - double last_time=0, now_time; + cur=0; + last_time=0; while(1) { + int retries = 0; struct seed_status *st; apr_status_t ret = apr_queue_pop(log_queue, (void**)&st); + while(ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_pop(log_queue, (void**)&st); + } if(ret != APR_SUCCESS || !st) break; if(st->status == MAPCACHE_STATUS_FINISHED) return NULL; if(st->status == MAPCACHE_STATUS_OK) { failed[cur]=0; n_metatiles_tot++; + if(st->nodata) { + n_nodata_tot++; + } if(!quiet) { struct mctimeval now; mapcache_gettimeofday(&now,NULL); @@ -674,20 +811,20 @@ } else { /* count how many errors and successes we have */ failed[cur]=1; - int i,nfailed=0,ntotal=0; + nfailed=0; + ntotal=0; if(failed_log) { fprintf(failed_log,"%d,%d,%d\n",st->x,st->y,st->z); } for(i=0; i=0) ntotal++; - if(failed[i]==1) nfailed++; + if(failed[i]==1) nfailed++; } ctx.log(&ctx, MAPCACHE_WARN, "failed to seed tile z%d,x%d,y%d:\n%s\n", st->z,st->x,st->y,st->msg); - double pct = ((double)nfailed / (double)ntotal) * 100; + pct = ((double)nfailed / (double)ntotal) * 100; if(pct > percent_failed_allowed) { ctx.log(&ctx, MAPCACHE_ERROR, "aborting seed as %.1f%% of the last %d requests failed\n", pct, FAIL_BACKLOG_COUNT); error_detected = 1; - return NULL; } } if(st->msg) free(st->msg); @@ -741,10 +878,18 @@ printf("usage: %s options\n",progname); while(seed_options[i].name) { - if(seed_options[i].has_arg==TRUE) { - printf("-%c|--%s [value]: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + if(seed_options[i].optch<256) { + if(seed_options[i].has_arg==TRUE) { + printf("-%c|--%s [value]: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + } else { + printf("-%c|--%s: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + } } else { - printf("-%c|--%s: %s\n",seed_options[i].optch,seed_options[i].name, seed_options[i].description); + if(seed_options[i].has_arg==TRUE) { + printf(" --%s [value]: %s\n",seed_options[i].name, seed_options[i].description); + } else { + printf(" --%s: %s\n",seed_options[i].name, seed_options[i].description); + } } i++; } @@ -762,9 +907,8 @@ /* initialize apr_getopt_t */ apr_getopt_t *opt; const char *configfile=NULL; - apr_thread_t **threads; - apr_thread_t *log_thread; - apr_threadattr_t *thread_attrs; + apr_thread_t **seed_threads; + apr_thread_t *log_thread,*feed_thread; const char *tileset_name=NULL; const char *tileset_transfer_name=NULL; const char *grid_name = NULL; @@ -781,8 +925,11 @@ int *metasizes = NULL;//[2]; int metax=-1,metay=-1; double *extent_array = NULL; + double thread_delay = 0.0; #ifdef USE_CLIPPERS + OGRFeatureH hFeature; + GEOSWKTReader *geoswktreader; const char *ogr_where = NULL; const char *ogr_layer = NULL; const char *ogr_sql = NULL; @@ -835,10 +982,10 @@ case 'i': if(!strcmp(optarg,"drill-down")) { iteration_mode = MAPCACHE_ITERATION_DEPTH_FIRST; - } else if(!strcmp(optarg,"level-by-level")) { + } else if(!strcmp(optarg,"level-by-level") || !strcmp(optarg, "scanline")) { iteration_mode = MAPCACHE_ITERATION_LEVEL_FIRST; } else { - return usage(argv[0],"invalid iteration mode, expecting \"drill-down\" or \"level-by-level\""); + return usage(argv[0],"invalid iteration mode, expecting \"drill-down\" or \"scanline\""); } break; case 'L': @@ -933,6 +1080,16 @@ } apr_table_set(argdimensions,dimkey,dimvalue); break; + case SEEDER_OPT_THREAD_DELAY: + thread_delay = strtod(optarg, NULL); + if(thread_delay < 0.0 ) + return usage(argv[0], "failed to parse thread-delay, expecting positive number of seconds"); + break; + case SEEDER_OPT_RATE_LIMIT: + rate_limit = (int)strtol(optarg, NULL, 10); + if(rate_limit <= 0 ) + return usage(argv[0], "failed to parse rate-limit, expecting positive number of tiles per seconds"); + break; #ifdef USE_CLIPPERS case 'd': ogr_datasource = optarg; @@ -976,6 +1133,7 @@ } if(ogr_datasource) { + int f=0; OGRDataSourceH hDS = NULL; OGRLayerH layer = NULL; OGRRegisterAll(); @@ -1023,21 +1181,20 @@ clippers = (const GEOSPreparedGeometry**)malloc(nClippers*sizeof(GEOSPreparedGeometry*)); - OGRFeatureH hFeature; - GEOSWKTReader *geoswktreader = GEOSWKTReader_create(); + geoswktreader = GEOSWKTReader_create(); OGR_L_ResetReading(layer); - extent = apr_palloc(ctx.pool,4*sizeof(mapcache_extent)); - int f=0; + extent = apr_palloc(ctx.pool,sizeof(mapcache_extent)); while( (hFeature = OGR_L_GetNextFeature(layer)) != NULL ) { + char *wkt; + GEOSGeometry *geosgeom; + OGREnvelope ogr_extent; OGRGeometryH geom = OGR_F_GetGeometryRef(hFeature); if(!geom || !OGR_G_IsValid(geom)) continue; - char *wkt; OGR_G_ExportToWkt(geom,&wkt); - GEOSGeometry *geosgeom = GEOSWKTReader_read(geoswktreader,wkt); + geosgeom = GEOSWKTReader_read(geoswktreader,wkt); free(wkt); clippers[f] = GEOSPrepare(geosgeom); //GEOSGeom_destroy(geosgeom); - OGREnvelope ogr_extent; OGR_G_GetEnvelope (geom, &ogr_extent); if(f == 0) { extent->minx = ogr_extent.MinX; @@ -1067,10 +1224,16 @@ if(!tileset) { return usage(argv[0], "tileset not found in configuration"); } - if(tileset->read_only) { - printf("tileset %s is read-only, switching it to read-write for seeding\n",tileset_name); + if(tileset->read_only && mode != MAPCACHE_CMD_TRANSFER) { + printf("tileset (%s) is read-only, switching it to read-write for seeding\n",tileset_name); tileset->read_only = 0; } + if( mode == MAPCACHE_CMD_TRANSFER ) { + if( tileset->read_only == 0 ) { + printf("switching tileset (%s) to read-only as we are in transfer mode\n",tileset_name); + } + tileset->read_only = 1; + } if( ! grid_name ) { grid_link = APR_ARRAY_IDX(tileset->grid_links,0,mapcache_grid_link*); } else { @@ -1086,6 +1249,38 @@ return usage(argv[0],"grid not configured for tileset"); } } +#ifdef USE_CLIPPERS + if(ogr_datasource) { + /* check that the provided ogr features are compatible with the grid units */ + if(grid_link->grid->unit == MAPCACHE_UNIT_DEGREES) { + if(extent->minx < -181.0 || + extent->maxx > 181.0 || + extent->miny < -91.0 || + extent->maxy > 91.0) { + printf("\n********************************************************************************\n" + "* WARNING!!!: you are seeding a grid in latlon degreees,\n" + "* but your provided OGR intersection features span (%f,%f,%f,%f).\n" + "* this seems like an error, you should be providing OGR features that\n" + "* are in the same projection as the grid you want to seed\n" + "********************************************************************************\n\n", + extent->minx, extent->miny, extent->maxx, extent->maxy); + } + } else { + if(extent->minx > -181.0 && + extent->maxx < 181.0 && + extent->miny > -91.0 && + extent->maxy < 91.0) { + printf("\n********************************************************************************\n" + "* WARNING!!!: you are seeding a grid that is not in latlon degreees,\n" + "* but your provided OGR intersection features span (%f,%f,%f,%f) which seem to be in degrees.\n" + "* this seems like an error, you should be providing OGR features that\n" + "* are in the same projection as the grid you want to seed\n" + "********************************************************************************\n\n", + extent->minx, extent->miny, extent->maxx, extent->maxy); + } + } + } +#endif if(iteration_mode == MAPCACHE_ITERATION_UNSET) { if(!strcmp(grid_link->grid->name,"g") || !strcmp(grid_link->grid->name,"WGS84") || !strcmp(grid_link->grid->name,"GoogleMapsCompatible")) { @@ -1105,7 +1300,7 @@ if(minzoomminz) minzoom = grid_link->minz; if(maxzoom>= grid_link->maxz) maxzoom = grid_link->maxz - 1; if(grid_link->outofzoom_strategy != MAPCACHE_OUTOFZOOM_NOTCONFIGURED && maxzoom > grid_link->max_cached_zoom) { - maxzoom = grid_link->max_cached_zoom; + return usage(argv[0],"requested maxzoom %d is higher than configured max-cached-zoom %d for grid",maxzoom,grid_link->max_cached_zoom); } /* adjust metasize */ @@ -1117,14 +1312,14 @@ /* ensure our metasize is a power of 2 in drill down mode */ if(iteration_mode == MAPCACHE_ITERATION_DEPTH_FIRST) { if(!isPowerOfTwo(tileset->metasize_x) || !isPowerOfTwo(tileset->metasize_y)) { - return usage(argv[0],"metatile size is not set to a power of two and iteration mode set to \"drill-down\", rerun with e.g -M 8,8, or force iteration mode to \"level-by-level\""); + return usage(argv[0],"metatile size is not set to a power of two and iteration mode set to \"drill-down\", rerun with e.g -M 8,8, or force iteration mode to \"scanline\""); } } if(cache_override) { mapcache_cache *co = mapcache_configuration_get_cache(cfg, cache_override); if(!co) { - return usage(argv[0], "overrided cache\"%s\" to not found in configuration", cache_override); + return usage(argv[0], "overridden cache\"%s\" not found in configuration", cache_override); } else { tileset->_cache = co; } @@ -1133,6 +1328,7 @@ } if (mode == MAPCACHE_CMD_TRANSFER) { + tileset->metasize_x = tileset->metasize_y = 1; if (!tileset_transfer_name) return usage(argv[0],"tileset where tiles should be transferred to not specified"); @@ -1182,75 +1378,55 @@ } /* validate the supplied dimensions */ - if (!apr_is_empty_array(tileset->dimensions) || tileset->timedimension) { + if (!apr_is_empty_array(tileset->dimensions)) { int i; const char *value; - dimensions = apr_table_make(ctx.pool,3); - if (!apr_is_empty_array(tileset->dimensions)) { - for(i=0; idimensions->nelts; i++) { - mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); - if((value = (char*)apr_table_get(argdimensions,dimension->name)) != NULL) { - char *tmpval = apr_pstrdup(ctx.pool,value); - int ok = dimension->validate(&ctx,dimension,&tmpval); - if(GC_HAS_ERROR(&ctx) || ok != MAPCACHE_SUCCESS ) { - return usage(argv[0],"failed to validate dimension"); - return 1; - } else { - /* validate may have changed the dimension value, so set that value into the dimensions table */ - apr_table_setn(dimensions,dimension->name,tmpval); - } - } else { - /* a dimension was not specified on the command line, add the default value */ - apr_table_setn(dimensions, dimension->name, dimension->default_value); - } + dimensions = apr_array_make(ctx.pool,tileset->dimensions->nelts,sizeof(mapcache_requested_dimension*)); + for(i=0; idimensions->nelts; i++) { + mapcache_dimension *dimension = APR_ARRAY_IDX(tileset->dimensions,i,mapcache_dimension*); + mapcache_requested_dimension *rdim = apr_pcalloc(ctx.pool,sizeof(mapcache_requested_dimension)); + rdim->dimension = dimension; + if((value = apr_table_get(argdimensions,dimension->name)) == NULL) { + value = dimension->default_value; + } + rdim->requested_value = apr_pstrdup(ctx.pool,value); + APR_ARRAY_PUSH(dimensions,mapcache_requested_dimension*)=rdim; + } + if(tileset->dimension_assembly_type != MAPCACHE_DIMENSION_ASSEMBLY_NONE) { + if(!tileset->store_dimension_assemblies) { + return usage(argv[0],"cannot seed a layer with dimension assembling and no caching of resulting assembly"); } - } - if(tileset->timedimension) { - if((value = (char*)apr_table_get(argdimensions,tileset->timedimension->key)) != NULL) { - apr_array_header_t *timedim_selected = mapcache_timedimension_get_entries_for_value(&ctx,tileset->timedimension, tileset, grid_link->grid, extent, value); - if(GC_HAS_ERROR(&ctx) || !timedim_selected) { - return usage(argv[0],"failed to validate time dimension"); - } - if(timedim_selected->nelts == 0) { - return usage(argv[0],"Time dimension %s=%s returns no configured entry",tileset->timedimension->key, - value); - } - if(timedim_selected->nelts > 1) { - return usage(argv[0],"Time dimension %s=%s returns more than 1 configured entries",tileset->timedimension->key, - value); - } - apr_table_set(dimensions,tileset->timedimension->key,APR_ARRAY_IDX(timedim_selected,0,char*)); - } else { - return usage(argv[0],"tileset references a TIME dimension, but none supplied on commandline. (hint: -D %s=",tileset->timedimension->key); - + if(tileset->metasize_x != 1 || tileset->metasize_y != 1) { + return usage(argv[0],"cannot seed a layer with dimension assembling and metatiling enabled (hint: rerun with -M 1,1 to disable metatiling"); } } } + + + if(nthreads == 0 && nprocesses == 0) { + nthreads = 1; + } + if(nthreads >= 1 && nprocesses >= 1) { + return usage(argv[0],"cannot set both nthreads and nprocesses"); + } { /* start the logging thread */ //create the queue where the seeding statuses will be put apr_threadattr_t *log_thread_attrs; - apr_queue_create(&log_queue,2,ctx.pool); + apr_queue_create(&log_queue,MAPCACHE_MAX(nthreads,nprocesses),ctx.pool); //start the rendering threads. apr_threadattr_create(&log_thread_attrs, ctx.pool); apr_thread_create(&log_thread, log_thread_attrs, log_thread_fn, NULL, ctx.pool); } - - if(nthreads == 0 && nprocesses == 0) { - nthreads = 1; - } - if(nthreads >= 1 && nprocesses >= 1) { - return usage(argv[0],"cannot set both nthreads and nprocesses"); - } + if(nprocesses > 1) { #ifdef USE_FORK key_t key; int i; pid_t *pids = malloc(nprocesses*sizeof(pid_t)); struct msqid_ds queue_ds; - ctx.threadlock = NULL; key = ftok(argv[0], 'B'); if ((msqid = msgget(key, 0644 | IPC_CREAT|S_IRUSR|S_IWUSR)) == -1) { return usage(argv[0],"failed to create sysv ipc message queue"); @@ -1285,7 +1461,7 @@ pids[i] = pid; } } - cmd_worker(); + feed_worker(); for(i=0; i 0) { + apr_sleep((int)(thread_delay * 1000000)); + } + apr_thread_create(&seed_threads[n], seed_thread_attrs, seed_thread, NULL, ctx.pool); } - cmd_worker(); + + + //the worker has finished generating the list of tiles to be seeded, now wait for the rendering threads to finish for(n=0; nstatus = MAPCACHE_STATUS_FINISHED; - apr_queue_push(log_queue,(void*)st); + ret = apr_queue_push(log_queue,(void*)st); + while (ret == APR_EINTR && retries<10) { + retries++; + ret = apr_queue_push(log_queue,(void*)st); + } apr_thread_join(&rv, log_thread); } @@ -1322,10 +1520,17 @@ struct mctimeval now_t; float duration; int ntilestot = n_metatiles_tot*tileset->metasize_x*tileset->metasize_y; + int nnodatatot = n_nodata_tot*tileset->metasize_x*tileset->metasize_y; mapcache_gettimeofday(&now_t,NULL); duration = ((now_t.tv_sec-starttime.tv_sec)*1000000+(now_t.tv_usec-starttime.tv_usec))/1000000.0; - printf("\nseeded %d metatiles (%d tiles) in %.1f seconds at %.1f tiles/sec\n",n_metatiles_tot, ntilestot, duration, ntilestot/duration); + printf("\nseeded %d metatiles (%d total tiles, %d non-empty tiles) in %.1f seconds at %.1f tiles/sec (%.1f non-empty tiles/sec)\n", + n_metatiles_tot, + ntilestot, + ntilestot-nnodatatot, + duration, + ntilestot/duration, + (ntilestot-nnodatatot)/duration); } else { if(!error_detected) { printf("0 tiles needed to be seeded, exiting\n"); @@ -1334,9 +1539,9 @@ apr_terminate(); if (error_detected > 0) { - exit(1); + exit(1); } - + return 0; } /* vim: ts=2 sts=2 et sw=2