diff options
author | Andrea Shepard <andrea@torproject.org> | 2012-10-30 14:39:14 -0700 |
---|---|---|
committer | Andrea Shepard <andrea@torproject.org> | 2012-10-30 14:39:14 -0700 |
commit | be371250305b0ac50c1fafda8acd13812894ab92 (patch) | |
tree | a3621a3f80a80d3a9139e0454bad3ee35c1f0000 /src/or/channel.c | |
parent | 03b32161bc37766864bcb099b76884099c89d837 (diff) | |
parent | 2b10e99eb093e1e5e699db355e0c54342fa693b5 (diff) | |
download | tor-be371250305b0ac50c1fafda8acd13812894ab92.tar.gz tor-be371250305b0ac50c1fafda8acd13812894ab92.zip |
Merge branch 'bsd_queue' of ssh://git-rw.torproject.org/nickm/tor
Diffstat (limited to 'src/or/channel.c')
-rw-r--r-- | src/or/channel.c | 349 |
1 files changed, 133 insertions, 216 deletions
diff --git a/src/or/channel.c b/src/or/channel.c index 5552d39f5b..cd5972f1ec 100644 --- a/src/or/channel.c +++ b/src/or/channel.c @@ -33,6 +33,7 @@ typedef struct cell_queue_entry_s cell_queue_entry_t; struct cell_queue_entry_s { + SIMPLEQ_ENTRY(cell_queue_entry_s) next; enum { CELL_QUEUE_FIXED, CELL_QUEUE_VAR, @@ -82,7 +83,37 @@ static uint64_t n_channels_allocated = 0; * If more than one channel exists, follow the next_with_same_id pointer * as a linked list. */ -static digestmap_t *channel_identity_map = NULL; +HT_HEAD(channel_idmap, channel_idmap_entry_s) channel_identity_map = + HT_INITIALIZER(); + +typedef struct channel_idmap_entry_s { + HT_ENTRY(channel_idmap_entry_s) node; + uint8_t digest[DIGEST_LEN]; + LIST_HEAD(channel_list_s, channel_s) channel_list; +} channel_idmap_entry_t; + +static INLINE unsigned +channel_idmap_hash(const channel_idmap_entry_t *ent) +{ + const unsigned *a = (const unsigned *)ent->digest; +#if SIZEOF_INT == 4 + return a[0] ^ a[1] ^ a[2] ^ a[3] ^ a[4]; +#elif SIZEOF_INT == 8 + return a[0] ^ a[1]; +#endif +} + +static INLINE int +channel_idmap_eq(const channel_idmap_entry_t *a, + const channel_idmap_entry_t *b) +{ + return tor_memeq(a->digest, b->digest, DIGEST_LEN); +} + +HT_PROTOTYPE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, + channel_idmap_eq); +HT_GENERATE(channel_idmap, channel_idmap_entry_s, node, channel_idmap_hash, + channel_idmap_eq, 0.5, tor_malloc, tor_realloc, tor_free_); static cell_queue_entry_t * cell_queue_entry_dup(cell_queue_entry_t *q); static void cell_queue_entry_free(cell_queue_entry_t *q, int handed_off); @@ -506,7 +537,7 @@ channel_listener_unregister(channel_listener_t *chan_l) static void channel_add_to_digest_map(channel_t *chan) { - channel_t *tmp; + channel_idmap_entry_t *ent, search; tor_assert(chan); @@ -518,23 +549,15 @@ channel_add_to_digest_map(channel_t *chan) /* Assert that there is a digest */ tor_assert(!tor_digest_is_zero(chan->identity_digest)); - /* Allocate the identity map if we have to */ - if (!channel_identity_map) channel_identity_map = digestmap_new(); - - /* Insert it */ - tmp = digestmap_set(channel_identity_map, - chan->identity_digest, - chan); - if (tmp) { - /* There already was one, this goes at the head of the list */ - chan->next_with_same_id = tmp; - chan->prev_with_same_id = NULL; - tmp->prev_with_same_id = chan; - } else { - /* First with this digest */ - chan->next_with_same_id = NULL; - chan->prev_with_same_id = NULL; + memcpy(search.digest, chan->identity_digest, DIGEST_LEN); + ent = HT_FIND(channel_idmap, &channel_identity_map, &search); + if (! ent) { + ent = tor_malloc(sizeof(channel_idmap_entry_t)); + memcpy(ent->digest, chan->identity_digest, DIGEST_LEN); + LIST_INIT(&ent->channel_list); + HT_INSERT(channel_idmap, &channel_identity_map, ent); } + LIST_INSERT_HEAD(&ent->channel_list, chan, next_with_same_id); log_debug(LD_CHANNEL, "Added channel %p (global ID " U64_FORMAT ") " @@ -554,13 +577,14 @@ channel_add_to_digest_map(channel_t *chan) static void channel_remove_from_digest_map(channel_t *chan) { - channel_t *tmp; + channel_idmap_entry_t *ent, search; tor_assert(chan); /* Assert that there is a digest */ tor_assert(!tor_digest_is_zero(chan->identity_digest)); +#if 0 /* Make sure we have a map */ if (!channel_identity_map) { /* @@ -585,66 +609,29 @@ channel_remove_from_digest_map(channel_t *chan) return; } +#endif - /* Look for it in the map */ - tmp = digestmap_get(channel_identity_map, chan->identity_digest); - if (tmp) { - /* Okay, it's here */ - /* Look for this channel */ - while (tmp && tmp != chan) { - tmp = tmp->next_with_same_id; - } + /* Pull it out of its list, wherever that list is */ + LIST_REMOVE(chan, next_with_same_id); - if (tmp == chan) { - /* Found it, good */ - if (chan->next_with_same_id) { - chan->next_with_same_id->prev_with_same_id = chan->prev_with_same_id; - } - /* else we're the tail of the list */ - if (chan->prev_with_same_id) { - /* We're not the head of the list, so we can *just* unlink */ - chan->prev_with_same_id->next_with_same_id = chan->next_with_same_id; - } else { - /* We're the head, so we have to point the digest map entry at our - * next if we have one, or remove it if we're also the tail */ - if (chan->next_with_same_id) { - digestmap_set(channel_identity_map, - chan->identity_digest, - chan->next_with_same_id); - } else { - digestmap_remove(channel_identity_map, - chan->identity_digest); - } - } + memcpy(search.digest, chan->identity_digest, DIGEST_LEN); + ent = HT_FIND(channel_idmap, &channel_identity_map, &search); - /* NULL out its next/prev pointers, and we're finished */ - chan->next_with_same_id = NULL; - chan->prev_with_same_id = NULL; + /* Look for it in the map */ + if (ent) { + /* Okay, it's here */ - log_debug(LD_CHANNEL, - "Removed channel %p (global ID " U64_FORMAT ") from " - "identity map in state %s (%d) with digest %s", - chan, U64_PRINTF_ARG(chan->global_identifier), - channel_state_to_string(chan->state), chan->state, - hex_str(chan->identity_digest, DIGEST_LEN)); - } else { - /* This is not good */ - log_warn(LD_BUG, - "Trying to remove channel %p (global ID " U64_FORMAT ") " - "with digest %s from identity map, but couldn't find it in " - "the list for that digest", - chan, U64_PRINTF_ARG(chan->global_identifier), - hex_str(chan->identity_digest, DIGEST_LEN)); - /* Unlink it and hope for the best */ - if (chan->next_with_same_id) { - chan->next_with_same_id->prev_with_same_id = chan->prev_with_same_id; - } - if (chan->prev_with_same_id) { - chan->prev_with_same_id->next_with_same_id = chan->next_with_same_id; - } - chan->next_with_same_id = NULL; - chan->prev_with_same_id = NULL; + if (LIST_EMPTY(&ent->channel_list)) { + HT_REMOVE(channel_idmap, &channel_identity_map, ent); + tor_free(ent); } + + log_debug(LD_CHANNEL, + "Removed channel %p (global ID " U64_FORMAT ") from " + "identity map in state %s (%d) with digest %s", + chan, U64_PRINTF_ARG(chan->global_identifier), + channel_state_to_string(chan->state), chan->state, + hex_str(chan->identity_digest, DIGEST_LEN)); } else { /* Shouldn't happen */ log_warn(LD_BUG, @@ -653,15 +640,6 @@ channel_remove_from_digest_map(channel_t *chan) "that digest", chan, U64_PRINTF_ARG(chan->global_identifier), hex_str(chan->identity_digest, DIGEST_LEN)); - /* Clear out its next/prev pointers */ - if (chan->next_with_same_id) { - chan->next_with_same_id->prev_with_same_id = chan->prev_with_same_id; - } - if (chan->prev_with_same_id) { - chan->prev_with_same_id->next_with_same_id = chan->next_with_same_id; - } - chan->next_with_same_id = NULL; - chan->prev_with_same_id = NULL; } } @@ -699,20 +677,21 @@ channel_find_by_global_id(uint64_t global_identifier) * * This function looks up a channel by the digest of its remote endpoint in * the channel digest map. It's possible that more than one channel to a - * given endpoint exists. Use channel_next_with_digest() and - * channel_prev_with_digest() to walk the list. + * given endpoint exists. Use channel_next_with_digest() to walk the list. */ channel_t * channel_find_by_remote_digest(const char *identity_digest) { channel_t *rv = NULL; + channel_idmap_entry_t *ent, search; tor_assert(identity_digest); - /* Search for it in the identity map */ - if (channel_identity_map) { - rv = digestmap_get(channel_identity_map, identity_digest); + memcpy(search.digest, identity_digest, DIGEST_LEN); + ent = HT_FIND(channel_idmap, &channel_identity_map, &search); + if (ent) { + rv = LIST_FIRST(&ent->channel_list); } return rv; @@ -730,22 +709,7 @@ channel_next_with_digest(channel_t *chan) { tor_assert(chan); - return chan->next_with_same_id; -} - -/** - * Get previous channel with digest - * - * This function takes a channel and finds the previos channel in the list - * with the same digest. - */ - -channel_t * -channel_prev_with_digest(channel_t *chan) -{ - tor_assert(chan); - - return chan->prev_with_same_id; + return LIST_NEXT(chan, next_with_same_id); } /** @@ -770,6 +734,13 @@ channel_init(channel_t *chan) /* Init next_circ_id */ chan->next_circ_id = crypto_rand_int(1 << 15); + /* Initialize queues. */ + SIMPLEQ_INIT(&chan->incoming_queue); + SIMPLEQ_INIT(&chan->outgoing_queue); + + /* Initialize list entries. */ + memset(&chan->next_with_same_id, 0, sizeof(chan->next_with_same_id)); + /* Timestamp it */ channel_timestamp_created(chan); } @@ -881,6 +852,7 @@ channel_listener_free(channel_listener_t *chan_l) static void channel_force_free(channel_t *chan) { + cell_queue_entry_t *cell, *cell_tmp; tor_assert(chan); log_debug(LD_CHANNEL, @@ -907,26 +879,16 @@ channel_force_free(channel_t *chan) } /* We might still have a cell queue; kill it */ - if (chan->incoming_queue) { - SMARTLIST_FOREACH_BEGIN(chan->incoming_queue, - cell_queue_entry_t *, q) { - cell_queue_entry_free(q, 0); - } SMARTLIST_FOREACH_END(q); - - smartlist_free(chan->incoming_queue); - chan->incoming_queue = NULL; + SIMPLEQ_FOREACH_SAFE(cell, &chan->incoming_queue, next, cell_tmp) { + cell_queue_entry_free(cell, 0); } + SIMPLEQ_INIT(&chan->incoming_queue); /* Outgoing cell queue is similar, but we can have to free packed cells */ - if (chan->outgoing_queue) { - SMARTLIST_FOREACH_BEGIN(chan->outgoing_queue, - cell_queue_entry_t *, q) { - cell_queue_entry_free(q, 0); - } SMARTLIST_FOREACH_END(q); - - smartlist_free(chan->outgoing_queue); - chan->outgoing_queue = NULL; + SIMPLEQ_FOREACH_SAFE(cell, &chan->outgoing_queue, next, cell_tmp) { + cell_queue_entry_free(cell, 0); } + SIMPLEQ_INIT(&chan->outgoing_queue); tor_free(chan); } @@ -1089,8 +1051,7 @@ channel_set_cell_handlers(channel_t *chan, chan->var_cell_handler = var_cell_handler; /* Re-run the queue if we have one and there's any reason to */ - if (chan->incoming_queue && - (smartlist_len(chan->incoming_queue) > 0) && + if (! SIMPLEQ_EMPTY(&chan->incoming_queue) && try_again && (chan->cell_handler || chan->var_cell_handler)) channel_process_cells(chan); @@ -1712,9 +1673,8 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) } /* Can we send it right out? If so, try */ - if (!(chan->outgoing_queue && - (smartlist_len(chan->outgoing_queue) > 0)) && - chan->state == CHANNEL_STATE_OPEN) { + if (SIMPLEQ_EMPTY(&chan->outgoing_queue) && + chan->state == CHANNEL_STATE_OPEN) { /* Pick the right write function for this cell type and save the result */ switch (q->type) { case CELL_QUEUE_FIXED: @@ -1750,14 +1710,12 @@ channel_write_cell_queue_entry(channel_t *chan, cell_queue_entry_t *q) if (!sent) { /* Not sent, queue it */ - if (!(chan->outgoing_queue)) - chan->outgoing_queue = smartlist_new(); /* * We have to copy the queue entry passed in, since the caller probably * used the stack. */ tmp = cell_queue_entry_dup(q); - smartlist_add(chan->outgoing_queue, tmp); + SIMPLEQ_INSERT_TAIL(&chan->outgoing_queue, tmp, next); /* Try to process the queue? */ if (chan->state == CHANNEL_STATE_OPEN) channel_flush_cells(chan); } @@ -1943,19 +1901,15 @@ channel_change_state(channel_t *chan, channel_state_t to_state) channel_do_open_actions(chan); /* Check for queued cells to process */ - if (chan->incoming_queue && - smartlist_len(chan->incoming_queue) > 0) + if (! SIMPLEQ_EMPTY(&chan->incoming_queue)) channel_process_cells(chan); - if (chan->outgoing_queue && - smartlist_len(chan->outgoing_queue) > 0) + if (! SIMPLEQ_EMPTY(&chan->outgoing_queue)) channel_flush_cells(chan); } else if (to_state == CHANNEL_STATE_CLOSED || to_state == CHANNEL_STATE_ERROR) { /* Assert that all queues are empty */ - tor_assert(!(chan->incoming_queue) || - smartlist_len(chan->incoming_queue) == 0); - tor_assert(!(chan->outgoing_queue) || - smartlist_len(chan->outgoing_queue) == 0); + tor_assert(SIMPLEQ_EMPTY(&chan->incoming_queue)); + tor_assert(SIMPLEQ_EMPTY(&chan->outgoing_queue)); } } @@ -2129,16 +2083,9 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, /* If we aren't in CHANNEL_STATE_OPEN, nothing goes through */ if (chan->state == CHANNEL_STATE_OPEN) { while ((unlimited || num_cells > flushed) && - (chan->outgoing_queue && - (smartlist_len(chan->outgoing_queue) > 0))) { - /* - * Ewww, smartlist_del_keeporder() is O(n) in list length; maybe a - * a linked list would make more sense for the queue. - */ - - /* Get the head of the queue */ - q = smartlist_get(chan->outgoing_queue, 0); - if (q) { + NULL != (q = SIMPLEQ_FIRST(&chan->outgoing_queue))) { + + if (1) { /* * Okay, we have a good queue entry, try to give it to the lower * layer. @@ -2223,26 +2170,17 @@ channel_flush_some_cells_from_outgoing_queue(channel_t *chan, cell_queue_entry_free(q, 0); q = NULL; } - } else { - /* This shouldn't happen; log and throw it away */ - log_info(LD_CHANNEL, - "Saw a NULL entry in the outgoing cell queue on channel %p " - "(global ID " U64_FORMAT "); this is definitely a bug.", - chan, U64_PRINTF_ARG(chan->global_identifier)); - /* q is already NULL, so we know to delete that queue entry */ - } - /* if q got NULLed out, we used it and should remove the queue entry */ - if (!q) smartlist_del_keeporder(chan->outgoing_queue, 0); - /* No cell removed from list, so we can't go on any further */ - else break; + /* if q got NULLed out, we used it and should remove the queue entry */ + if (!q) SIMPLEQ_REMOVE_HEAD(&chan->outgoing_queue, next); + /* No cell removed from list, so we can't go on any further */ + else break; + } } } /* Did we drain the queue? */ - if (!(chan->outgoing_queue) || - smartlist_len(chan->outgoing_queue) == 0) { - /* Timestamp it */ + if (SIMPLEQ_EMPTY(&chan->outgoing_queue)) { channel_timestamp_drained(chan); } @@ -2276,8 +2214,8 @@ channel_more_to_flush(channel_t *chan) tor_assert(chan); /* Check if we have any queued */ - if (chan->incoming_queue && - smartlist_len(chan->incoming_queue) > 0) return 1; + if (! SIMPLEQ_EMPTY(&chan->incoming_queue)) + return 1; /* Check if any circuits would like to queue some */ if (circuitmux_num_cells(chan->cmux) > 0) return 1; @@ -2470,8 +2408,7 @@ channel_listener_queue_incoming(channel_listener_t *listener, void channel_process_cells(channel_t *chan) { - smartlist_t *queue; - int putback = 0; + cell_queue_entry_t *q; tor_assert(chan); tor_assert(chan->state == CHANNEL_STATE_CLOSING || chan->state == CHANNEL_STATE_MAINT || @@ -2485,24 +2422,21 @@ channel_process_cells(channel_t *chan) if (!(chan->cell_handler || chan->var_cell_handler)) return; /* Nothing we can do if we have no cells */ - if (!(chan->incoming_queue)) return; + if (SIMPLEQ_EMPTY(&chan->incoming_queue)) return; - queue = chan->incoming_queue; - chan->incoming_queue = NULL; /* * Process cells until we're done or find one we have no current handler * for. */ - SMARTLIST_FOREACH_BEGIN(queue, cell_queue_entry_t *, q) { + while (NULL != (q = SIMPLEQ_FIRST(&chan->incoming_queue))) { tor_assert(q); tor_assert(q->type == CELL_QUEUE_FIXED || q->type == CELL_QUEUE_VAR); - if (putback) { - smartlist_add(chan->incoming_queue, q); - } else if (q->type == CELL_QUEUE_FIXED && + if (q->type == CELL_QUEUE_FIXED && chan->cell_handler) { /* Handle a fixed-length cell */ + SIMPLEQ_REMOVE_HEAD(&chan->incoming_queue, next); tor_assert(q->u.fixed.cell); log_debug(LD_CHANNEL, "Processing incoming cell_t %p for channel %p (global ID " @@ -2514,6 +2448,7 @@ channel_process_cells(channel_t *chan) } else if (q->type == CELL_QUEUE_VAR && chan->var_cell_handler) { /* Handle a variable-length cell */ + SIMPLEQ_REMOVE_HEAD(&chan->incoming_queue, next); tor_assert(q->u.var.var_cell); log_debug(LD_CHANNEL, "Processing incoming var_cell_t %p for channel %p (global ID " @@ -2524,15 +2459,10 @@ channel_process_cells(channel_t *chan) tor_free(q); } else { /* Can't handle this one */ - if (!chan->incoming_queue) - chan->incoming_queue = smartlist_new(); - smartlist_add(chan->incoming_queue, q); - putback = 1; + break; } - } SMARTLIST_FOREACH_END(q); + } - /* If the list is empty, free it */ - smartlist_free(queue); } /** @@ -2554,15 +2484,9 @@ channel_queue_cell(channel_t *chan, cell_t *cell) /* Do we need to queue it, or can we just call the handler right away? */ if (!(chan->cell_handler)) need_to_queue = 1; - if (chan->incoming_queue && - (smartlist_len(chan->incoming_queue) > 0)) + if (! SIMPLEQ_EMPTY(&chan->incoming_queue)) need_to_queue = 1; - /* If we need to queue and have no queue, create one */ - if (need_to_queue && !(chan->incoming_queue)) { - chan->incoming_queue = smartlist_new(); - } - /* Timestamp for receiving */ channel_timestamp_recv(chan); @@ -2580,14 +2504,13 @@ channel_queue_cell(channel_t *chan, cell_t *cell) chan->cell_handler(chan, cell); } else { /* Otherwise queue it and then process the queue if possible. */ - tor_assert(chan->incoming_queue); q = cell_queue_entry_new_fixed(cell); log_debug(LD_CHANNEL, "Queueing incoming cell_t %p for channel %p " "(global ID " U64_FORMAT ")", cell, chan, U64_PRINTF_ARG(chan->global_identifier)); - smartlist_add(chan->incoming_queue, q); + SIMPLEQ_INSERT_TAIL(&chan->incoming_queue, q, next); if (chan->cell_handler || chan->var_cell_handler) { channel_process_cells(chan); @@ -2614,15 +2537,9 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) /* Do we need to queue it, or can we just call the handler right away? */ if (!(chan->var_cell_handler)) need_to_queue = 1; - if (chan->incoming_queue && - (smartlist_len(chan->incoming_queue) > 0)) + if (! SIMPLEQ_EMPTY(&chan->incoming_queue)) need_to_queue = 1; - /* If we need to queue and have no queue, create one */ - if (need_to_queue && !(chan->incoming_queue)) { - chan->incoming_queue = smartlist_new(); - } - /* Timestamp for receiving */ channel_timestamp_recv(chan); @@ -2640,14 +2557,13 @@ channel_queue_var_cell(channel_t *chan, var_cell_t *var_cell) chan->var_cell_handler(chan, var_cell); } else { /* Otherwise queue it and then process the queue if possible. */ - tor_assert(chan->incoming_queue); q = cell_queue_entry_new_var(var_cell); log_debug(LD_CHANNEL, "Queueing incoming var_cell_t %p for channel %p " "(global ID " U64_FORMAT ")", var_cell, chan, U64_PRINTF_ARG(chan->global_identifier)); - smartlist_add(chan->incoming_queue, q); + SIMPLEQ_INSERT_TAIL(&chan->incoming_queue, q, next); if (chan->cell_handler || chan->var_cell_handler) { channel_process_cells(chan); @@ -2939,13 +2855,10 @@ channel_free_all(void) } /* Now free channel_identity_map */ - if (channel_identity_map) { - log_debug(LD_CHANNEL, - "Freeing channel_identity_map"); - /* Geez, anything still left over just won't die ... let it leak then */ - digestmap_free(channel_identity_map, NULL); - channel_identity_map = NULL; - } + log_debug(LD_CHANNEL, + "Freeing channel_identity_map"); + /* Geez, anything still left over just won't die ... let it leak then */ + HT_CLEAR(channel_idmap, &channel_identity_map); log_debug(LD_CHANNEL, "Done cleaning up after channels"); @@ -3052,12 +2965,6 @@ channel_get_for_extend(const char *digest, tor_assert(msg_out); tor_assert(launch_out); - if (!channel_identity_map) { - *msg_out = "Router not connected (nothing is). Connecting."; - *launch_out = 1; - return NULL; - } - chan = channel_find_by_remote_digest(digest); /* Walk the list, unrefing the old one and refing the new at each @@ -3177,6 +3084,19 @@ channel_listener_describe_transport(channel_listener_t *chan_l) } /** + * Return the number of entries in <b>queue</b> + */ +static int +chan_cell_queue_len(const chan_cell_queue_t *queue) +{ + int r = 0; + cell_queue_entry_t *cell; + SIMPLEQ_FOREACH(cell, queue, next) + ++r; + return r; +} + +/** * Dump channel statistics * * Dump statistics for one channel to the log @@ -3293,10 +3213,8 @@ channel_dump_statistics(channel_t *chan, int severity) " * Channel " U64_FORMAT " has %d queued incoming cells" " and %d queued outgoing cells", U64_PRINTF_ARG(chan->global_identifier), - (chan->incoming_queue != NULL) ? - smartlist_len(chan->incoming_queue) : 0, - (chan->outgoing_queue != NULL) ? - smartlist_len(chan->outgoing_queue) : 0); + chan_cell_queue_len(&chan->incoming_queue), + chan_cell_queue_len(&chan->outgoing_queue)); /* Describe circuits */ log(severity, LD_GENERAL, @@ -3562,8 +3480,7 @@ channel_has_queued_writes(channel_t *chan) tor_assert(chan); tor_assert(chan->has_queued_writes); - if (chan->outgoing_queue && - smartlist_len(chan->outgoing_queue) > 0) { + if (! SIMPLEQ_EMPTY(&chan->outgoing_queue)) { has_writes = 1; } else { /* Check with the lower layer */ |