net/xfrm/xfrm_policy.c

   1 /*
   2  * xfrm_policy.c
   3  *
   4  * Changes:
   5  *      Mitsuru KANDA @USAGI
   6  *      Kazunori MIYAZAWA @USAGI
   7  *      Kunihiro Ishiguro <kunihiro@ipinfusion.com>
   8  *              IPv6 support
   9  *      Kazunori MIYAZAWA @USAGI
  10  *      YOSHIFUJI Hideaki
  11  *              Split up af-specific portion
  12  *      Derek Atkins <derek@ihtfp.com>          Add the post_input processor
  13  *
  14  */
  15
  16 #include <linux/err.h>
  17 #include <linux/slab.h>
  18 #include <linux/kmod.h>
  19 #include <linux/list.h>
  20 #include <linux/spinlock.h>
  21 #include <linux/workqueue.h>
  22 #include <linux/notifier.h>
  23 #include <linux/netdevice.h>
  24 #include <linux/netfilter.h>
  25 #include <linux/module.h>
  26 #include <linux/cache.h>
  27 #include <linux/audit.h>
  28 #include <net/dst.h>
  29 #include <net/xfrm.h>
  30 #include <net/ip.h>
  31 #ifdef CONFIG_XFRM_STATISTICS
  32 #include <net/snmp.h>
  33 #endif
  34
  35 #include "xfrm_hash.h"
  36
  37 int sysctl_xfrm_larval_drop __read_mostly = 1;
  38
  39 #ifdef CONFIG_XFRM_STATISTICS
  40 DEFINE_SNMP_STAT(struct linux_xfrm_mib, xfrm_statistics) __read_mostly;
  41 EXPORT_SYMBOL(xfrm_statistics);
  42 #endif
  43
  44 DEFINE_MUTEX(xfrm_cfg_mutex);
  45 EXPORT_SYMBOL(xfrm_cfg_mutex);
  46
  47 static DEFINE_RWLOCK(xfrm_policy_lock);
  48
  49 unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
  50 EXPORT_SYMBOL(xfrm_policy_count);
  51
  52 static DEFINE_RWLOCK(xfrm_policy_afinfo_lock);
  53 static struct xfrm_policy_afinfo *xfrm_policy_afinfo[NPROTO];
  54
  55 static struct kmem_cache *xfrm_dst_cache __read_mostly;
  56
  57 static HLIST_HEAD(xfrm_policy_gc_list);
  58 static DEFINE_SPINLOCK(xfrm_policy_gc_lock);
  59
  60 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family);
  61 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo);
  62 static void xfrm_init_pmtu(struct dst_entry *dst);
  63
  64 static inline int
  65 __xfrm4_selector_match(struct xfrm_selector *sel, struct flowi *fl)
  66 {
  67         return  addr_match(&fl->fl4_dst, &sel->daddr, sel->prefixlen_d) &&
  68                 addr_match(&fl->fl4_src, &sel->saddr, sel->prefixlen_s) &&
  69                 !((xfrm_flowi_dport(fl) ^ sel->dport) & sel->dport_mask) &&
  70                 !((xfrm_flowi_sport(fl) ^ sel->sport) & sel->sport_mask) &&
  71                 (fl->proto == sel->proto || !sel->proto) &&
  72                 (fl->oif == sel->ifindex || !sel->ifindex);
  73 }
  74
  75 static inline int
  76 __xfrm6_selector_match(struct xfrm_selector *sel, struct flowi *fl)
  77 {
  78         return  addr_match(&fl->fl6_dst, &sel->daddr, sel->prefixlen_d) &&
  79                 addr_match(&fl->fl6_src, &sel->saddr, sel->prefixlen_s) &&
  80                 !((xfrm_flowi_dport(fl) ^ sel->dport) & sel->dport_mask) &&
  81                 !((xfrm_flowi_sport(fl) ^ sel->sport) & sel->sport_mask) &&
  82                 (fl->proto == sel->proto || !sel->proto) &&
  83                 (fl->oif == sel->ifindex || !sel->ifindex);
  84 }
  85
  86 int xfrm_selector_match(struct xfrm_selector *sel, struct flowi *fl,
  87                     unsigned short family)
  88 {
  89         switch (family) {
  90         case AF_INET:
  91                 return __xfrm4_selector_match(sel, fl);
  92         case AF_INET6:
  93                 return __xfrm6_selector_match(sel, fl);
  94         }
  95         return 0;
  96 }
  97
  98 static inline struct dst_entry *__xfrm_dst_lookup(int tos,
  99                                                   xfrm_address_t *saddr,
 100                                                   xfrm_address_t *daddr,
 101                                                   int family)
 102 {
 103         struct xfrm_policy_afinfo *afinfo;
 104         struct dst_entry *dst;
 105
 106         afinfo = xfrm_policy_get_afinfo(family);
 107         if (unlikely(afinfo == NULL))
 108                 return ERR_PTR(-EAFNOSUPPORT);
 109
 110         dst = afinfo->dst_lookup(tos, saddr, daddr);
 111
 112         xfrm_policy_put_afinfo(afinfo);
 113
 114         return dst;
 115 }
 116
 117 static inline struct dst_entry *xfrm_dst_lookup(struct xfrm_state *x, int tos,
 118                                                 xfrm_address_t *prev_saddr,
 119                                                 xfrm_address_t *prev_daddr,
 120                                                 int family)
 121 {
 122         xfrm_address_t *saddr = &x->props.saddr;
 123         xfrm_address_t *daddr = &x->id.daddr;
 124         struct dst_entry *dst;
 125
 126         if (x->type->flags & XFRM_TYPE_LOCAL_COADDR) {
 127                 saddr = x->coaddr;
 128                 daddr = prev_daddr;
 129         }
 130         if (x->type->flags & XFRM_TYPE_REMOTE_COADDR) {
 131                 saddr = prev_saddr;
 132                 daddr = x->coaddr;
 133         }
 134
 135         dst = __xfrm_dst_lookup(tos, saddr, daddr, family);
 136
 137         if (!IS_ERR(dst)) {
 138                 if (prev_saddr != saddr)
 139                         memcpy(prev_saddr, saddr,  sizeof(*prev_saddr));
 140                 if (prev_daddr != daddr)
 141                         memcpy(prev_daddr, daddr,  sizeof(*prev_daddr));
 142         }
 143
 144         return dst;
 145 }
 146
 147 static inline unsigned long make_jiffies(long secs)
 148 {
 149         if (secs >= (MAX_SCHEDULE_TIMEOUT-1)/HZ)
 150                 return MAX_SCHEDULE_TIMEOUT-1;
 151         else
 152                 return secs*HZ;
 153 }
 154
 155 static void xfrm_policy_timer(unsigned long data)
 156 {
 157         struct xfrm_policy *xp = (struct xfrm_policy*)data;
 158         unsigned long now = get_seconds();
 159         long next = LONG_MAX;
 160         int warn = 0;
 161         int dir;
 162
 163         read_lock(&xp->lock);
 164
 165         if (xp->walk.dead)
 166                 goto out;
 167
 168         dir = xfrm_policy_id2dir(xp->index);
 169
 170         if (xp->lft.hard_add_expires_seconds) {
 171                 long tmo = xp->lft.hard_add_expires_seconds +
 172                         xp->curlft.add_time - now;
 173                 if (tmo <= 0)
 174                         goto expired;
 175                 if (tmo < next)
 176                         next = tmo;
 177         }
 178         if (xp->lft.hard_use_expires_seconds) {
 179                 long tmo = xp->lft.hard_use_expires_seconds +
 180                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
 181                 if (tmo <= 0)
 182                         goto expired;
 183                 if (tmo < next)
 184                         next = tmo;
 185         }
 186         if (xp->lft.soft_add_expires_seconds) {
 187                 long tmo = xp->lft.soft_add_expires_seconds +
 188                         xp->curlft.add_time - now;
 189                 if (tmo <= 0) {
 190                         warn = 1;
 191                         tmo = XFRM_KM_TIMEOUT;
 192                 }
 193                 if (tmo < next)
 194                         next = tmo;
 195         }
 196         if (xp->lft.soft_use_expires_seconds) {
 197                 long tmo = xp->lft.soft_use_expires_seconds +
 198                         (xp->curlft.use_time ? : xp->curlft.add_time) - now;
 199                 if (tmo <= 0) {
 200                         warn = 1;
 201                         tmo = XFRM_KM_TIMEOUT;
 202                 }
 203                 if (tmo < next)
 204                         next = tmo;
 205         }
 206
 207         if (warn)
 208                 km_policy_expired(xp, dir, 0, 0);
 209         if (next != LONG_MAX &&
 210             !mod_timer(&xp->timer, jiffies + make_jiffies(next)))
 211                 xfrm_pol_hold(xp);
 212
 213 out:
 214         read_unlock(&xp->lock);
 215         xfrm_pol_put(xp);
 216         return;
 217
 218 expired:
 219         read_unlock(&xp->lock);
 220         if (!xfrm_policy_delete(xp, dir))
 221                 km_policy_expired(xp, dir, 1, 0);
 222         xfrm_pol_put(xp);
 223 }
 224
 225
 226 /* Allocate xfrm_policy. Not used here, it is supposed to be used by pfkeyv2
 227  * SPD calls.
 228  */
 229
 230 struct xfrm_policy *xfrm_policy_alloc(struct net *net, gfp_t gfp)
 231 {
 232         struct xfrm_policy *policy;
 233
 234         policy = kzalloc(sizeof(struct xfrm_policy), gfp);
 235
 236         if (policy) {
 237                 write_pnet(&policy->xp_net, net);
 238                 INIT_LIST_HEAD(&policy->walk.all);
 239                 INIT_HLIST_NODE(&policy->bydst);
 240                 INIT_HLIST_NODE(&policy->byidx);
 241                 rwlock_init(&policy->lock);
 242                 atomic_set(&policy->refcnt, 1);
 243                 setup_timer(&policy->timer, xfrm_policy_timer,
 244                                 (unsigned long)policy);
 245         }
 246         return policy;
 247 }
 248 EXPORT_SYMBOL(xfrm_policy_alloc);
 249
 250 /* Destroy xfrm_policy: descendant resources must be released to this moment. */
 251
 252 void xfrm_policy_destroy(struct xfrm_policy *policy)
 253 {
 254         BUG_ON(!policy->walk.dead);
 255
 256         BUG_ON(policy->bundles);
 257
 258         if (del_timer(&policy->timer))
 259                 BUG();
 260
 261         security_xfrm_policy_free(policy->security);
 262         kfree(policy);
 263 }
 264 EXPORT_SYMBOL(xfrm_policy_destroy);
 265
 266 static void xfrm_policy_gc_kill(struct xfrm_policy *policy)
 267 {
 268         struct dst_entry *dst;
 269
 270         while ((dst = policy->bundles) != NULL) {
 271                 policy->bundles = dst->next;
 272                 dst_free(dst);
 273         }
 274
 275         if (del_timer(&policy->timer))
 276                 atomic_dec(&policy->refcnt);
 277
 278         if (atomic_read(&policy->refcnt) > 1)
 279                 flow_cache_flush();
 280
 281         xfrm_pol_put(policy);
 282 }
 283
 284 static void xfrm_policy_gc_task(struct work_struct *work)
 285 {
 286         struct xfrm_policy *policy;
 287         struct hlist_node *entry, *tmp;
 288         struct hlist_head gc_list;
 289
 290         spin_lock_bh(&xfrm_policy_gc_lock);
 291         gc_list.first = xfrm_policy_gc_list.first;
 292         INIT_HLIST_HEAD(&xfrm_policy_gc_list);
 293         spin_unlock_bh(&xfrm_policy_gc_lock);
 294
 295         hlist_for_each_entry_safe(policy, entry, tmp, &gc_list, bydst)
 296                 xfrm_policy_gc_kill(policy);
 297 }
 298 static DECLARE_WORK(xfrm_policy_gc_work, xfrm_policy_gc_task);
 299
 300 /* Rule must be locked. Release descentant resources, announce
 301  * entry dead. The rule must be unlinked from lists to the moment.
 302  */
 303
 304 static void xfrm_policy_kill(struct xfrm_policy *policy)
 305 {
 306         int dead;
 307
 308         write_lock_bh(&policy->lock);
 309         dead = policy->walk.dead;
 310         policy->walk.dead = 1;
 311         write_unlock_bh(&policy->lock);
 312
 313         if (unlikely(dead)) {
 314                 WARN_ON(1);
 315                 return;
 316         }
 317
 318         spin_lock_bh(&xfrm_policy_gc_lock);
 319         hlist_add_head(&policy->bydst, &xfrm_policy_gc_list);
 320         spin_unlock_bh(&xfrm_policy_gc_lock);
 321
 322         schedule_work(&xfrm_policy_gc_work);
 323 }
 324
 325 struct xfrm_policy_hash {
 326         struct hlist_head       *table;
 327         unsigned int            hmask;
 328 };
 329
 330 static struct xfrm_policy_hash xfrm_policy_bydst[XFRM_POLICY_MAX*2] __read_mostly;
 331 static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
 332
 333 static inline unsigned int idx_hash(u32 index)
 334 {
 335         return __idx_hash(index, init_net.xfrm.policy_idx_hmask);
 336 }
 337
 338 static struct hlist_head *policy_hash_bysel(struct xfrm_selector *sel, unsigned short family, int dir)
 339 {
 340         unsigned int hmask = xfrm_policy_bydst[dir].hmask;
 341         unsigned int hash = __sel_hash(sel, family, hmask);
 342
 343         return (hash == hmask + 1 ?
 344                 &init_net.xfrm.policy_inexact[dir] :
 345                 xfrm_policy_bydst[dir].table + hash);
 346 }
 347
 348 static struct hlist_head *policy_hash_direct(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, int dir)
 349 {
 350         unsigned int hmask = xfrm_policy_bydst[dir].hmask;
 351         unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
 352
 353         return xfrm_policy_bydst[dir].table + hash;
 354 }
 355
 356 static void xfrm_dst_hash_transfer(struct hlist_head *list,
 357                                    struct hlist_head *ndsttable,
 358                                    unsigned int nhashmask)
 359 {
 360         struct hlist_node *entry, *tmp, *entry0 = NULL;
 361         struct xfrm_policy *pol;
 362         unsigned int h0 = 0;
 363
 364 redo:
 365         hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {
 366                 unsigned int h;
 367
 368                 h = __addr_hash(&pol->selector.daddr, &pol->selector.saddr,
 369                                 pol->family, nhashmask);
 370                 if (!entry0) {
 371                         hlist_del(entry);
 372                         hlist_add_head(&pol->bydst, ndsttable+h);
 373                         h0 = h;
 374                 } else {
 375                         if (h != h0)
 376                                 continue;
 377                         hlist_del(entry);
 378                         hlist_add_after(entry0, &pol->bydst);
 379                 }
 380                 entry0 = entry;
 381         }
 382         if (!hlist_empty(list)) {
 383                 entry0 = NULL;
 384                 goto redo;
 385         }
 386 }
 387
 388 static void xfrm_idx_hash_transfer(struct hlist_head *list,
 389                                    struct hlist_head *nidxtable,
 390                                    unsigned int nhashmask)
 391 {
 392         struct hlist_node *entry, *tmp;
 393         struct xfrm_policy *pol;
 394
 395         hlist_for_each_entry_safe(pol, entry, tmp, list, byidx) {
 396                 unsigned int h;
 397
 398                 h = __idx_hash(pol->index, nhashmask);
 399                 hlist_add_head(&pol->byidx, nidxtable+h);
 400         }
 401 }
 402
 403 static unsigned long xfrm_new_hash_mask(unsigned int old_hmask)
 404 {
 405         return ((old_hmask + 1) << 1) - 1;
 406 }
 407
 408 static void xfrm_bydst_resize(int dir)
 409 {
 410         unsigned int hmask = xfrm_policy_bydst[dir].hmask;
 411         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 412         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 413         struct hlist_head *odst = xfrm_policy_bydst[dir].table;
 414         struct hlist_head *ndst = xfrm_hash_alloc(nsize);
 415         int i;
 416
 417         if (!ndst)
 418                 return;
 419
 420         write_lock_bh(&xfrm_policy_lock);
 421
 422         for (i = hmask; i >= 0; i--)
 423                 xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
 424
 425         xfrm_policy_bydst[dir].table = ndst;
 426         xfrm_policy_bydst[dir].hmask = nhashmask;
 427
 428         write_unlock_bh(&xfrm_policy_lock);
 429
 430         xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
 431 }
 432
 433 static void xfrm_byidx_resize(int total)
 434 {
 435         unsigned int hmask = init_net.xfrm.policy_idx_hmask;
 436         unsigned int nhashmask = xfrm_new_hash_mask(hmask);
 437         unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
 438         struct hlist_head *oidx = init_net.xfrm.policy_byidx;
 439         struct hlist_head *nidx = xfrm_hash_alloc(nsize);
 440         int i;
 441
 442         if (!nidx)
 443                 return;
 444
 445         write_lock_bh(&xfrm_policy_lock);
 446
 447         for (i = hmask; i >= 0; i--)
 448                 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
 449
 450         init_net.xfrm.policy_byidx = nidx;
 451         init_net.xfrm.policy_idx_hmask = nhashmask;
 452
 453         write_unlock_bh(&xfrm_policy_lock);
 454
 455         xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
 456 }
 457
 458 static inline int xfrm_bydst_should_resize(int dir, int *total)
 459 {
 460         unsigned int cnt = xfrm_policy_count[dir];
 461         unsigned int hmask = xfrm_policy_bydst[dir].hmask;
 462
 463         if (total)
 464                 *total += cnt;
 465
 466         if ((hmask + 1) < xfrm_policy_hashmax &&
 467             cnt > hmask)
 468                 return 1;
 469
 470         return 0;
 471 }
 472
 473 static inline int xfrm_byidx_should_resize(int total)
 474 {
 475         unsigned int hmask = init_net.xfrm.policy_idx_hmask;
 476
 477         if ((hmask + 1) < xfrm_policy_hashmax &&
 478             total > hmask)
 479                 return 1;
 480
 481         return 0;
 482 }
 483
 484 void xfrm_spd_getinfo(struct xfrmk_spdinfo *si)
 485 {
 486         read_lock_bh(&xfrm_policy_lock);
 487         si->incnt = xfrm_policy_count[XFRM_POLICY_IN];
 488         si->outcnt = xfrm_policy_count[XFRM_POLICY_OUT];
 489         si->fwdcnt = xfrm_policy_count[XFRM_POLICY_FWD];
 490         si->inscnt = xfrm_policy_count[XFRM_POLICY_IN+XFRM_POLICY_MAX];
 491         si->outscnt = xfrm_policy_count[XFRM_POLICY_OUT+XFRM_POLICY_MAX];
 492         si->fwdscnt = xfrm_policy_count[XFRM_POLICY_FWD+XFRM_POLICY_MAX];
 493         si->spdhcnt = init_net.xfrm.policy_idx_hmask;
 494         si->spdhmcnt = xfrm_policy_hashmax;
 495         read_unlock_bh(&xfrm_policy_lock);
 496 }
 497 EXPORT_SYMBOL(xfrm_spd_getinfo);
 498
 499 static DEFINE_MUTEX(hash_resize_mutex);
 500 static void xfrm_hash_resize(struct work_struct *__unused)
 501 {
 502         int dir, total;
 503
 504         mutex_lock(&hash_resize_mutex);
 505
 506         total = 0;
 507         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
 508                 if (xfrm_bydst_should_resize(dir, &total))
 509                         xfrm_bydst_resize(dir);
 510         }
 511         if (xfrm_byidx_should_resize(total))
 512                 xfrm_byidx_resize(total);
 513
 514         mutex_unlock(&hash_resize_mutex);
 515 }
 516
 517 static DECLARE_WORK(xfrm_hash_work, xfrm_hash_resize);
 518
 519 /* Generate new index... KAME seems to generate them ordered by cost
 520  * of an absolute inpredictability of ordering of rules. This will not pass. */
 521 static u32 xfrm_gen_index(int dir)
 522 {
 523         static u32 idx_generator;
 524
 525         for (;;) {
 526                 struct hlist_node *entry;
 527                 struct hlist_head *list;
 528                 struct xfrm_policy *p;
 529                 u32 idx;
 530                 int found;
 531
 532                 idx = (idx_generator | dir);
 533                 idx_generator += 8;
 534                 if (idx == 0)
 535                         idx = 8;
 536                 list = init_net.xfrm.policy_byidx + idx_hash(idx);
 537                 found = 0;
 538                 hlist_for_each_entry(p, entry, list, byidx) {
 539                         if (p->index == idx) {
 540                                 found = 1;
 541                                 break;
 542                         }
 543                 }
 544                 if (!found)
 545                         return idx;
 546         }
 547 }
 548
 549 static inline int selector_cmp(struct xfrm_selector *s1, struct xfrm_selector *s2)
 550 {
 551         u32 *p1 = (u32 *) s1;
 552         u32 *p2 = (u32 *) s2;
 553         int len = sizeof(struct xfrm_selector) / sizeof(u32);
 554         int i;
 555
 556         for (i = 0; i < len; i++) {
 557                 if (p1[i] != p2[i])
 558                         return 1;
 559         }
 560
 561         return 0;
 562 }
 563
 564 int xfrm_policy_insert(int dir, struct xfrm_policy *policy, int excl)
 565 {
 566         struct xfrm_policy *pol;
 567         struct xfrm_policy *delpol;
 568         struct hlist_head *chain;
 569         struct hlist_node *entry, *newpos;
 570         struct dst_entry *gc_list;
 571
 572         write_lock_bh(&xfrm_policy_lock);
 573         chain = policy_hash_bysel(&policy->selector, policy->family, dir);
 574         delpol = NULL;
 575         newpos = NULL;
 576         hlist_for_each_entry(pol, entry, chain, bydst) {
 577                 if (pol->type == policy->type &&
 578                     !selector_cmp(&pol->selector, &policy->selector) &&
 579                     xfrm_sec_ctx_match(pol->security, policy->security) &&
 580                     !WARN_ON(delpol)) {
 581                         if (excl) {
 582                                 write_unlock_bh(&xfrm_policy_lock);
 583                                 return -EEXIST;
 584                         }
 585                         delpol = pol;
 586                         if (policy->priority > pol->priority)
 587                                 continue;
 588                 } else if (policy->priority >= pol->priority) {
 589                         newpos = &pol->bydst;
 590                         continue;
 591                 }
 592                 if (delpol)
 593                         break;
 594         }
 595         if (newpos)
 596                 hlist_add_after(newpos, &policy->bydst);
 597         else
 598                 hlist_add_head(&policy->bydst, chain);
 599         xfrm_pol_hold(policy);
 600         xfrm_policy_count[dir]++;
 601         atomic_inc(&flow_cache_genid);
 602         if (delpol) {
 603                 hlist_del(&delpol->bydst);
 604                 hlist_del(&delpol->byidx);
 605                 list_del(&delpol->walk.all);
 606                 xfrm_policy_count[dir]--;
 607         }
 608         policy->index = delpol ? delpol->index : xfrm_gen_index(dir);
 609         hlist_add_head(&policy->byidx, init_net.xfrm.policy_byidx+idx_hash(policy->index));
 610         policy->curlft.add_time = get_seconds();
 611         policy->curlft.use_time = 0;
 612         if (!mod_timer(&policy->timer, jiffies + HZ))
 613                 xfrm_pol_hold(policy);
 614         list_add(&policy->walk.all, &init_net.xfrm.policy_all);
 615         write_unlock_bh(&xfrm_policy_lock);
 616
 617         if (delpol)
 618                 xfrm_policy_kill(delpol);
 619         else if (xfrm_bydst_should_resize(dir, NULL))
 620                 schedule_work(&xfrm_hash_work);
 621
 622         read_lock_bh(&xfrm_policy_lock);
 623         gc_list = NULL;
 624         entry = &policy->bydst;
 625         hlist_for_each_entry_continue(policy, entry, bydst) {
 626                 struct dst_entry *dst;
 627
 628                 write_lock(&policy->lock);
 629                 dst = policy->bundles;
 630                 if (dst) {
 631                         struct dst_entry *tail = dst;
 632                         while (tail->next)
 633                                 tail = tail->next;
 634                         tail->next = gc_list;
 635                         gc_list = dst;
 636
 637                         policy->bundles = NULL;
 638                 }
 639                 write_unlock(&policy->lock);
 640         }
 641         read_unlock_bh(&xfrm_policy_lock);
 642
 643         while (gc_list) {
 644                 struct dst_entry *dst = gc_list;
 645
 646                 gc_list = dst->next;
 647                 dst_free(dst);
 648         }
 649
 650         return 0;
 651 }
 652 EXPORT_SYMBOL(xfrm_policy_insert);
 653
 654 struct xfrm_policy *xfrm_policy_bysel_ctx(u8 type, int dir,
 655                                           struct xfrm_selector *sel,
 656                                           struct xfrm_sec_ctx *ctx, int delete,
 657                                           int *err)
 658 {
 659         struct xfrm_policy *pol, *ret;
 660         struct hlist_head *chain;
 661         struct hlist_node *entry;
 662
 663         *err = 0;
 664         write_lock_bh(&xfrm_policy_lock);
 665         chain = policy_hash_bysel(sel, sel->family, dir);
 666         ret = NULL;
 667         hlist_for_each_entry(pol, entry, chain, bydst) {
 668                 if (pol->type == type &&
 669                     !selector_cmp(sel, &pol->selector) &&
 670                     xfrm_sec_ctx_match(ctx, pol->security)) {
 671                         xfrm_pol_hold(pol);
 672                         if (delete) {
 673                                 *err = security_xfrm_policy_delete(
 674                                                                 pol->security);
 675                                 if (*err) {
 676                                         write_unlock_bh(&xfrm_policy_lock);
 677                                         return pol;
 678                                 }
 679                                 hlist_del(&pol->bydst);
 680                                 hlist_del(&pol->byidx);
 681                                 list_del(&pol->walk.all);
 682                                 xfrm_policy_count[dir]--;
 683                         }
 684                         ret = pol;
 685                         break;
 686                 }
 687         }
 688         write_unlock_bh(&xfrm_policy_lock);
 689
 690         if (ret && delete) {
 691                 atomic_inc(&flow_cache_genid);
 692                 xfrm_policy_kill(ret);
 693         }
 694         return ret;
 695 }
 696 EXPORT_SYMBOL(xfrm_policy_bysel_ctx);
 697
 698 struct xfrm_policy *xfrm_policy_byid(u8 type, int dir, u32 id, int delete,
 699                                      int *err)
 700 {
 701         struct xfrm_policy *pol, *ret;
 702         struct hlist_head *chain;
 703         struct hlist_node *entry;
 704
 705         *err = -ENOENT;
 706         if (xfrm_policy_id2dir(id) != dir)
 707                 return NULL;
 708
 709         *err = 0;
 710         write_lock_bh(&xfrm_policy_lock);
 711         chain = init_net.xfrm.policy_byidx + idx_hash(id);
 712         ret = NULL;
 713         hlist_for_each_entry(pol, entry, chain, byidx) {
 714                 if (pol->type == type && pol->index == id) {
 715                         xfrm_pol_hold(pol);
 716                         if (delete) {
 717                                 *err = security_xfrm_policy_delete(
 718                                                                 pol->security);
 719                                 if (*err) {
 720                                         write_unlock_bh(&xfrm_policy_lock);
 721                                         return pol;
 722                                 }
 723                                 hlist_del(&pol->bydst);
 724                                 hlist_del(&pol->byidx);
 725                                 list_del(&pol->walk.all);
 726                                 xfrm_policy_count[dir]--;
 727                         }
 728                         ret = pol;
 729                         break;
 730                 }
 731         }
 732         write_unlock_bh(&xfrm_policy_lock);
 733
 734         if (ret && delete) {
 735                 atomic_inc(&flow_cache_genid);
 736                 xfrm_policy_kill(ret);
 737         }
 738         return ret;
 739 }
 740 EXPORT_SYMBOL(xfrm_policy_byid);
 741
 742 #ifdef CONFIG_SECURITY_NETWORK_XFRM
 743 static inline int
 744 xfrm_policy_flush_secctx_check(u8 type, struct xfrm_audit *audit_info)
 745 {
 746         int dir, err = 0;
 747
 748         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 749                 struct xfrm_policy *pol;
 750                 struct hlist_node *entry;
 751                 int i;
 752
 753                 hlist_for_each_entry(pol, entry,
 754                                      &init_net.xfrm.policy_inexact[dir], bydst) {
 755                         if (pol->type != type)
 756                                 continue;
 757                         err = security_xfrm_policy_delete(pol->security);
 758                         if (err) {
 759                                 xfrm_audit_policy_delete(pol, 0,
 760                                                          audit_info->loginuid,
 761                                                          audit_info->sessionid,
 762                                                          audit_info->secid);
 763                                 return err;
 764                         }
 765                 }
 766                 for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
 767                         hlist_for_each_entry(pol, entry,
 768                                              xfrm_policy_bydst[dir].table + i,
 769                                              bydst) {
 770                                 if (pol->type != type)
 771                                         continue;
 772                                 err = security_xfrm_policy_delete(
 773                                                                 pol->security);
 774                                 if (err) {
 775                                         xfrm_audit_policy_delete(pol, 0,
 776                                                         audit_info->loginuid,
 777                                                         audit_info->sessionid,
 778                                                         audit_info->secid);
 779                                         return err;
 780                                 }
 781                         }
 782                 }
 783         }
 784         return err;
 785 }
 786 #else
 787 static inline int
 788 xfrm_policy_flush_secctx_check(u8 type, struct xfrm_audit *audit_info)
 789 {
 790         return 0;
 791 }
 792 #endif
 793
 794 int xfrm_policy_flush(u8 type, struct xfrm_audit *audit_info)
 795 {
 796         int dir, err = 0;
 797
 798         write_lock_bh(&xfrm_policy_lock);
 799
 800         err = xfrm_policy_flush_secctx_check(type, audit_info);
 801         if (err)
 802                 goto out;
 803
 804         for (dir = 0; dir < XFRM_POLICY_MAX; dir++) {
 805                 struct xfrm_policy *pol;
 806                 struct hlist_node *entry;
 807                 int i, killed;
 808
 809                 killed = 0;
 810         again1:
 811                 hlist_for_each_entry(pol, entry,
 812                                      &init_net.xfrm.policy_inexact[dir], bydst) {
 813                         if (pol->type != type)
 814                                 continue;
 815                         hlist_del(&pol->bydst);
 816                         hlist_del(&pol->byidx);
 817                         write_unlock_bh(&xfrm_policy_lock);
 818
 819                         xfrm_audit_policy_delete(pol, 1, audit_info->loginuid,
 820                                                  audit_info->sessionid,
 821                                                  audit_info->secid);
 822
 823                         xfrm_policy_kill(pol);
 824                         killed++;
 825
 826                         write_lock_bh(&xfrm_policy_lock);
 827                         goto again1;
 828                 }
 829
 830                 for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
 831         again2:
 832                         hlist_for_each_entry(pol, entry,
 833                                              xfrm_policy_bydst[dir].table + i,
 834                                              bydst) {
 835                                 if (pol->type != type)
 836                                         continue;
 837                                 hlist_del(&pol->bydst);
 838                                 hlist_del(&pol->byidx);
 839                                 list_del(&pol->walk.all);
 840                                 write_unlock_bh(&xfrm_policy_lock);
 841
 842                                 xfrm_audit_policy_delete(pol, 1,
 843                                                          audit_info->loginuid,
 844                                                          audit_info->sessionid,
 845                                                          audit_info->secid);
 846                                 xfrm_policy_kill(pol);
 847                                 killed++;
 848
 849                                 write_lock_bh(&xfrm_policy_lock);
 850                                 goto again2;
 851                         }
 852                 }
 853
 854                 xfrm_policy_count[dir] -= killed;
 855         }
 856         atomic_inc(&flow_cache_genid);
 857 out:
 858         write_unlock_bh(&xfrm_policy_lock);
 859         return err;
 860 }
 861 EXPORT_SYMBOL(xfrm_policy_flush);
 862
 863 int xfrm_policy_walk(struct xfrm_policy_walk *walk,
 864                      int (*func)(struct xfrm_policy *, int, int, void*),
 865                      void *data)
 866 {
 867         struct xfrm_policy *pol;
 868         struct xfrm_policy_walk_entry *x;
 869         int error = 0;
 870
 871         if (walk->type >= XFRM_POLICY_TYPE_MAX &&
 872             walk->type != XFRM_POLICY_TYPE_ANY)
 873                 return -EINVAL;
 874
 875         if (list_empty(&walk->walk.all) && walk->seq != 0)
 876                 return 0;
 877
 878         write_lock_bh(&xfrm_policy_lock);
 879         if (list_empty(&walk->walk.all))
 880                 x = list_first_entry(&init_net.xfrm.policy_all, struct xfrm_policy_walk_entry, all);
 881         else
 882                 x = list_entry(&walk->walk.all, struct xfrm_policy_walk_entry, all);
 883         list_for_each_entry_from(x, &init_net.xfrm.policy_all, all) {
 884                 if (x->dead)
 885                         continue;
 886                 pol = container_of(x, struct xfrm_policy, walk);
 887                 if (walk->type != XFRM_POLICY_TYPE_ANY &&
 888                     walk->type != pol->type)
 889                         continue;
 890                 error = func(pol, xfrm_policy_id2dir(pol->index),
 891                              walk->seq, data);
 892                 if (error) {
 893                         list_move_tail(&walk->walk.all, &x->all);
 894                         goto out;
 895                 }
 896                 walk->seq++;
 897         }
 898         if (walk->seq == 0) {
 899                 error = -ENOENT;
 900                 goto out;
 901         }
 902         list_del_init(&walk->walk.all);
 903 out:
 904         write_unlock_bh(&xfrm_policy_lock);
 905         return error;
 906 }
 907 EXPORT_SYMBOL(xfrm_policy_walk);
 908
 909 void xfrm_policy_walk_init(struct xfrm_policy_walk *walk, u8 type)
 910 {
 911         INIT_LIST_HEAD(&walk->walk.all);
 912         walk->walk.dead = 1;
 913         walk->type = type;
 914         walk->seq = 0;
 915 }
 916 EXPORT_SYMBOL(xfrm_policy_walk_init);
 917
 918 void xfrm_policy_walk_done(struct xfrm_policy_walk *walk)
 919 {
 920         if (list_empty(&walk->walk.all))
 921                 return;
 922
 923         write_lock_bh(&xfrm_policy_lock);
 924         list_del(&walk->walk.all);
 925         write_unlock_bh(&xfrm_policy_lock);
 926 }
 927 EXPORT_SYMBOL(xfrm_policy_walk_done);
 928
 929 /*
 930  * Find policy to apply to this flow.
 931  *
 932  * Returns 0 if policy found, else an -errno.
 933  */
 934 static int xfrm_policy_match(struct xfrm_policy *pol, struct flowi *fl,
 935                              u8 type, u16 family, int dir)
 936 {
 937         struct xfrm_selector *sel = &pol->selector;
 938         int match, ret = -ESRCH;
 939
 940         if (pol->family != family ||
 941             pol->type != type)
 942                 return ret;
 943
 944         match = xfrm_selector_match(sel, fl, family);
 945         if (match)
 946                 ret = security_xfrm_policy_lookup(pol->security, fl->secid,
 947                                                   dir);
 948
 949         return ret;
 950 }
 951
 952 static struct xfrm_policy *xfrm_policy_lookup_bytype(u8 type, struct flowi *fl,
 953                                                      u16 family, u8 dir)
 954 {
 955         int err;
 956         struct xfrm_policy *pol, *ret;
 957         xfrm_address_t *daddr, *saddr;
 958         struct hlist_node *entry;
 959         struct hlist_head *chain;
 960         u32 priority = ~0U;
 961
 962         daddr = xfrm_flowi_daddr(fl, family);
 963         saddr = xfrm_flowi_saddr(fl, family);
 964         if (unlikely(!daddr || !saddr))
 965                 return NULL;
 966
 967         read_lock_bh(&xfrm_policy_lock);
 968         chain = policy_hash_direct(daddr, saddr, family, dir);
 969         ret = NULL;
 970         hlist_for_each_entry(pol, entry, chain, bydst) {
 971                 err = xfrm_policy_match(pol, fl, type, family, dir);
 972                 if (err) {
 973                         if (err == -ESRCH)
 974                                 continue;
 975                         else {
 976                                 ret = ERR_PTR(err);
 977                                 goto fail;
 978                         }
 979                 } else {
 980                         ret = pol;
 981                         priority = ret->priority;
 982                         break;
 983                 }
 984         }
 985         chain = &init_net.xfrm.policy_inexact[dir];
 986         hlist_for_each_entry(pol, entry, chain, bydst) {
 987                 err = xfrm_policy_match(pol, fl, type, family, dir);
 988                 if (err) {
 989                         if (err == -ESRCH)
 990                                 continue;
 991                         else {
 992                                 ret = ERR_PTR(err);
 993                                 goto fail;
 994                         }
 995                 } else if (pol->priority < priority) {
 996                         ret = pol;
 997                         break;
 998                 }
 999         }
1000         if (ret)
1001                 xfrm_pol_hold(ret);
1002 fail:
1003         read_unlock_bh(&xfrm_policy_lock);
1004
1005         return ret;
1006 }
1007
1008 static int xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
1009                                void **objp, atomic_t **obj_refp)
1010 {
1011         struct xfrm_policy *pol;
1012         int err = 0;
1013
1014 #ifdef CONFIG_XFRM_SUB_POLICY
1015         pol = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_SUB, fl, family, dir);
1016         if (IS_ERR(pol)) {
1017                 err = PTR_ERR(pol);
1018                 pol = NULL;
1019         }
1020         if (pol || err)
1021                 goto end;
1022 #endif
1023         pol = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN, fl, family, dir);
1024         if (IS_ERR(pol)) {
1025                 err = PTR_ERR(pol);
1026                 pol = NULL;
1027         }
1028 #ifdef CONFIG_XFRM_SUB_POLICY
1029 end:
1030 #endif
1031         if ((*objp = (void *) pol) != NULL)
1032                 *obj_refp = &pol->refcnt;
1033         return err;
1034 }
1035
1036 static inline int policy_to_flow_dir(int dir)
1037 {
1038         if (XFRM_POLICY_IN == FLOW_DIR_IN &&
1039             XFRM_POLICY_OUT == FLOW_DIR_OUT &&
1040             XFRM_POLICY_FWD == FLOW_DIR_FWD)
1041                 return dir;
1042         switch (dir) {
1043         default:
1044         case XFRM_POLICY_IN:
1045                 return FLOW_DIR_IN;
1046         case XFRM_POLICY_OUT:
1047                 return FLOW_DIR_OUT;
1048         case XFRM_POLICY_FWD:
1049                 return FLOW_DIR_FWD;
1050         }
1051 }
1052
1053 static struct xfrm_policy *xfrm_sk_policy_lookup(struct sock *sk, int dir, struct flowi *fl)
1054 {
1055         struct xfrm_policy *pol;
1056
1057         read_lock_bh(&xfrm_policy_lock);
1058         if ((pol = sk->sk_policy[dir]) != NULL) {
1059                 int match = xfrm_selector_match(&pol->selector, fl,
1060                                                 sk->sk_family);
1061                 int err = 0;
1062
1063                 if (match) {
1064                         err = security_xfrm_policy_lookup(pol->security,
1065                                                       fl->secid,
1066                                                       policy_to_flow_dir(dir));
1067                         if (!err)
1068                                 xfrm_pol_hold(pol);
1069                         else if (err == -ESRCH)
1070                                 pol = NULL;
1071                         else
1072                                 pol = ERR_PTR(err);
1073                 } else
1074                         pol = NULL;
1075         }
1076         read_unlock_bh(&xfrm_policy_lock);
1077         return pol;
1078 }
1079
1080 static void __xfrm_policy_link(struct xfrm_policy *pol, int dir)
1081 {
1082         struct hlist_head *chain = policy_hash_bysel(&pol->selector,
1083                                                      pol->family, dir);
1084
1085         list_add(&pol->walk.all, &init_net.xfrm.policy_all);
1086         hlist_add_head(&pol->bydst, chain);
1087         hlist_add_head(&pol->byidx, init_net.xfrm.policy_byidx+idx_hash(pol->index));
1088         xfrm_policy_count[dir]++;
1089         xfrm_pol_hold(pol);
1090
1091         if (xfrm_bydst_should_resize(dir, NULL))
1092                 schedule_work(&xfrm_hash_work);
1093 }
1094
1095 static struct xfrm_policy *__xfrm_policy_unlink(struct xfrm_policy *pol,
1096                                                 int dir)
1097 {
1098         if (hlist_unhashed(&pol->bydst))
1099                 return NULL;
1100
1101         hlist_del(&pol->bydst);
1102         hlist_del(&pol->byidx);
1103         list_del(&pol->walk.all);
1104         xfrm_policy_count[dir]--;
1105
1106         return pol;
1107 }
1108
1109 int xfrm_policy_delete(struct xfrm_policy *pol, int dir)
1110 {
1111         write_lock_bh(&xfrm_policy_lock);
1112         pol = __xfrm_policy_unlink(pol, dir);
1113         write_unlock_bh(&xfrm_policy_lock);
1114         if (pol) {
1115                 if (dir < XFRM_POLICY_MAX)
1116                         atomic_inc(&flow_cache_genid);
1117                 xfrm_policy_kill(pol);
1118                 return 0;
1119         }
1120         return -ENOENT;
1121 }
1122 EXPORT_SYMBOL(xfrm_policy_delete);
1123
1124 int xfrm_sk_policy_insert(struct sock *sk, int dir, struct xfrm_policy *pol)
1125 {
1126         struct xfrm_policy *old_pol;
1127
1128 #ifdef CONFIG_XFRM_SUB_POLICY
1129         if (pol && pol->type != XFRM_POLICY_TYPE_MAIN)
1130                 return -EINVAL;
1131 #endif
1132
1133         write_lock_bh(&xfrm_policy_lock);
1134         old_pol = sk->sk_policy[dir];
1135         sk->sk_policy[dir] = pol;
1136         if (pol) {
1137                 pol->curlft.add_time = get_seconds();
1138                 pol->index = xfrm_gen_index(XFRM_POLICY_MAX+dir);
1139                 __xfrm_policy_link(pol, XFRM_POLICY_MAX+dir);
1140         }
1141         if (old_pol)
1142                 __xfrm_policy_unlink(old_pol, XFRM_POLICY_MAX+dir);
1143         write_unlock_bh(&xfrm_policy_lock);
1144
1145         if (old_pol) {
1146                 xfrm_policy_kill(old_pol);
1147         }
1148         return 0;
1149 }
1150
1151 static struct xfrm_policy *clone_policy(struct xfrm_policy *old, int dir)
1152 {
1153         struct xfrm_policy *newp = xfrm_policy_alloc(xp_net(old), GFP_ATOMIC);
1154
1155         if (newp) {
1156                 newp->selector = old->selector;
1157                 if (security_xfrm_policy_clone(old->security,
1158                                                &newp->security)) {
1159                         kfree(newp);
1160                         return NULL;  /* ENOMEM */
1161                 }
1162                 newp->lft = old->lft;
1163                 newp->curlft = old->curlft;
1164                 newp->action = old->action;
1165                 newp->flags = old->flags;
1166                 newp->xfrm_nr = old->xfrm_nr;
1167                 newp->index = old->index;
1168                 newp->type = old->type;
1169                 memcpy(newp->xfrm_vec, old->xfrm_vec,
1170                        newp->xfrm_nr*sizeof(struct xfrm_tmpl));
1171                 write_lock_bh(&xfrm_policy_lock);
1172                 __xfrm_policy_link(newp, XFRM_POLICY_MAX+dir);
1173                 write_unlock_bh(&xfrm_policy_lock);
1174                 xfrm_pol_put(newp);
1175         }
1176         return newp;
1177 }
1178
1179 int __xfrm_sk_clone_policy(struct sock *sk)
1180 {
1181         struct xfrm_policy *p0 = sk->sk_policy[0],
1182                            *p1 = sk->sk_policy[1];
1183
1184         sk->sk_policy[0] = sk->sk_policy[1] = NULL;
1185         if (p0 && (sk->sk_policy[0] = clone_policy(p0, 0)) == NULL)
1186                 return -ENOMEM;
1187         if (p1 && (sk->sk_policy[1] = clone_policy(p1, 1)) == NULL)
1188                 return -ENOMEM;
1189         return 0;
1190 }
1191
1192 static int
1193 xfrm_get_saddr(xfrm_address_t *local, xfrm_address_t *remote,
1194                unsigned short family)
1195 {
1196         int err;
1197         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1198
1199         if (unlikely(afinfo == NULL))
1200                 return -EINVAL;
1201         err = afinfo->get_saddr(local, remote);
1202         xfrm_policy_put_afinfo(afinfo);
1203         return err;
1204 }
1205
1206 /* Resolve list of templates for the flow, given policy. */
1207
1208 static int
1209 xfrm_tmpl_resolve_one(struct xfrm_policy *policy, struct flowi *fl,
1210                       struct xfrm_state **xfrm,
1211                       unsigned short family)
1212 {
1213         int nx;
1214         int i, error;
1215         xfrm_address_t *daddr = xfrm_flowi_daddr(fl, family);
1216         xfrm_address_t *saddr = xfrm_flowi_saddr(fl, family);
1217         xfrm_address_t tmp;
1218
1219         for (nx=0, i = 0; i < policy->xfrm_nr; i++) {
1220                 struct xfrm_state *x;
1221                 xfrm_address_t *remote = daddr;
1222                 xfrm_address_t *local  = saddr;
1223                 struct xfrm_tmpl *tmpl = &policy->xfrm_vec[i];
1224
1225                 if (tmpl->mode == XFRM_MODE_TUNNEL ||
1226                     tmpl->mode == XFRM_MODE_BEET) {
1227                         remote = &tmpl->id.daddr;
1228                         local = &tmpl->saddr;
1229                         family = tmpl->encap_family;
1230                         if (xfrm_addr_any(local, family)) {
1231                                 error = xfrm_get_saddr(&tmp, remote, family);
1232                                 if (error)
1233                                         goto fail;
1234                                 local = &tmp;
1235                         }
1236                 }
1237
1238                 x = xfrm_state_find(remote, local, fl, tmpl, policy, &error, family);
1239
1240                 if (x && x->km.state == XFRM_STATE_VALID) {
1241                         xfrm[nx++] = x;
1242                         daddr = remote;
1243                         saddr = local;
1244                         continue;
1245                 }
1246                 if (x) {
1247                         error = (x->km.state == XFRM_STATE_ERROR ?
1248                                  -EINVAL : -EAGAIN);
1249                         xfrm_state_put(x);
1250                 }
1251                 else if (error == -ESRCH)
1252                         error = -EAGAIN;
1253
1254                 if (!tmpl->optional)
1255                         goto fail;
1256         }
1257         return nx;
1258
1259 fail:
1260         for (nx--; nx>=0; nx--)
1261                 xfrm_state_put(xfrm[nx]);
1262         return error;
1263 }
1264
1265 static int
1266 xfrm_tmpl_resolve(struct xfrm_policy **pols, int npols, struct flowi *fl,
1267                   struct xfrm_state **xfrm,
1268                   unsigned short family)
1269 {
1270         struct xfrm_state *tp[XFRM_MAX_DEPTH];
1271         struct xfrm_state **tpp = (npols > 1) ? tp : xfrm;
1272         int cnx = 0;
1273         int error;
1274         int ret;
1275         int i;
1276
1277         for (i = 0; i < npols; i++) {
1278                 if (cnx + pols[i]->xfrm_nr >= XFRM_MAX_DEPTH) {
1279                         error = -ENOBUFS;
1280                         goto fail;
1281                 }
1282
1283                 ret = xfrm_tmpl_resolve_one(pols[i], fl, &tpp[cnx], family);
1284                 if (ret < 0) {
1285                         error = ret;
1286                         goto fail;
1287                 } else
1288                         cnx += ret;
1289         }
1290
1291         /* found states are sorted for outbound processing */
1292         if (npols > 1)
1293                 xfrm_state_sort(xfrm, tpp, cnx, family);
1294
1295         return cnx;
1296
1297  fail:
1298         for (cnx--; cnx>=0; cnx--)
1299                 xfrm_state_put(tpp[cnx]);
1300         return error;
1301
1302 }
1303
1304 /* Check that the bundle accepts the flow and its components are
1305  * still valid.
1306  */
1307
1308 static struct dst_entry *
1309 xfrm_find_bundle(struct flowi *fl, struct xfrm_policy *policy, unsigned short family)
1310 {
1311         struct dst_entry *x;
1312         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1313         if (unlikely(afinfo == NULL))
1314                 return ERR_PTR(-EINVAL);
1315         x = afinfo->find_bundle(fl, policy);
1316         xfrm_policy_put_afinfo(afinfo);
1317         return x;
1318 }
1319
1320 static inline int xfrm_get_tos(struct flowi *fl, int family)
1321 {
1322         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1323         int tos;
1324
1325         if (!afinfo)
1326                 return -EINVAL;
1327
1328         tos = afinfo->get_tos(fl);
1329
1330         xfrm_policy_put_afinfo(afinfo);
1331
1332         return tos;
1333 }
1334
1335 static inline struct xfrm_dst *xfrm_alloc_dst(int family)
1336 {
1337         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1338         struct xfrm_dst *xdst;
1339
1340         if (!afinfo)
1341                 return ERR_PTR(-EINVAL);
1342
1343         xdst = dst_alloc(afinfo->dst_ops) ?: ERR_PTR(-ENOBUFS);
1344
1345         xfrm_policy_put_afinfo(afinfo);
1346
1347         return xdst;
1348 }
1349
1350 static inline int xfrm_init_path(struct xfrm_dst *path, struct dst_entry *dst,
1351                                  int nfheader_len)
1352 {
1353         struct xfrm_policy_afinfo *afinfo =
1354                 xfrm_policy_get_afinfo(dst->ops->family);
1355         int err;
1356
1357         if (!afinfo)
1358                 return -EINVAL;
1359
1360         err = afinfo->init_path(path, dst, nfheader_len);
1361
1362         xfrm_policy_put_afinfo(afinfo);
1363
1364         return err;
1365 }
1366
1367 static inline int xfrm_fill_dst(struct xfrm_dst *xdst, struct net_device *dev)
1368 {
1369         struct xfrm_policy_afinfo *afinfo =
1370                 xfrm_policy_get_afinfo(xdst->u.dst.ops->family);
1371         int err;
1372
1373         if (!afinfo)
1374                 return -EINVAL;
1375
1376         err = afinfo->fill_dst(xdst, dev);
1377
1378         xfrm_policy_put_afinfo(afinfo);
1379
1380         return err;
1381 }
1382
1383 /* Allocate chain of dst_entry's, attach known xfrm's, calculate
1384  * all the metrics... Shortly, bundle a bundle.
1385  */
1386
1387 static struct dst_entry *xfrm_bundle_create(struct xfrm_policy *policy,
1388                                             struct xfrm_state **xfrm, int nx,
1389                                             struct flowi *fl,
1390                                             struct dst_entry *dst)
1391 {
1392         unsigned long now = jiffies;
1393         struct net_device *dev;
1394         struct dst_entry *dst_prev = NULL;
1395         struct dst_entry *dst0 = NULL;
1396         int i = 0;
1397         int err;
1398         int header_len = 0;
1399         int nfheader_len = 0;
1400         int trailer_len = 0;
1401         int tos;
1402         int family = policy->selector.family;
1403         xfrm_address_t saddr, daddr;
1404
1405         xfrm_flowi_addr_get(fl, &saddr, &daddr, family);
1406
1407         tos = xfrm_get_tos(fl, family);
1408         err = tos;
1409         if (tos < 0)
1410                 goto put_states;
1411
1412         dst_hold(dst);
1413
1414         for (; i < nx; i++) {
1415                 struct xfrm_dst *xdst = xfrm_alloc_dst(family);
1416                 struct dst_entry *dst1 = &xdst->u.dst;
1417
1418                 err = PTR_ERR(xdst);
1419                 if (IS_ERR(xdst)) {
1420                         dst_release(dst);
1421                         goto put_states;
1422                 }
1423
1424                 if (!dst_prev)
1425                         dst0 = dst1;
1426                 else {
1427                         dst_prev->child = dst_clone(dst1);
1428                         dst1->flags |= DST_NOHASH;
1429                 }
1430
1431                 xdst->route = dst;
1432                 memcpy(&dst1->metrics, &dst->metrics, sizeof(dst->metrics));
1433
1434                 if (xfrm[i]->props.mode != XFRM_MODE_TRANSPORT) {
1435                         family = xfrm[i]->props.family;
1436                         dst = xfrm_dst_lookup(xfrm[i], tos, &saddr, &daddr,
1437                                               family);
1438                         err = PTR_ERR(dst);
1439                         if (IS_ERR(dst))
1440                                 goto put_states;
1441                 } else
1442                         dst_hold(dst);
1443
1444                 dst1->xfrm = xfrm[i];
1445                 xdst->genid = xfrm[i]->genid;
1446
1447                 dst1->obsolete = -1;
1448                 dst1->flags |= DST_HOST;
1449                 dst1->lastuse = now;
1450
1451                 dst1->input = dst_discard;
1452                 dst1->output = xfrm[i]->outer_mode->afinfo->output;
1453
1454                 dst1->next = dst_prev;
1455                 dst_prev = dst1;
1456
1457                 header_len += xfrm[i]->props.header_len;
1458                 if (xfrm[i]->type->flags & XFRM_TYPE_NON_FRAGMENT)
1459                         nfheader_len += xfrm[i]->props.header_len;
1460                 trailer_len += xfrm[i]->props.trailer_len;
1461         }
1462
1463         dst_prev->child = dst;
1464         dst0->path = dst;
1465
1466         err = -ENODEV;
1467         dev = dst->dev;
1468         if (!dev)
1469                 goto free_dst;
1470
1471         /* Copy neighbout for reachability confirmation */
1472         dst0->neighbour = neigh_clone(dst->neighbour);
1473
1474         xfrm_init_path((struct xfrm_dst *)dst0, dst, nfheader_len);
1475         xfrm_init_pmtu(dst_prev);
1476
1477         for (dst_prev = dst0; dst_prev != dst; dst_prev = dst_prev->child) {
1478                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst_prev;
1479
1480                 err = xfrm_fill_dst(xdst, dev);
1481                 if (err)
1482                         goto free_dst;
1483
1484                 dst_prev->header_len = header_len;
1485                 dst_prev->trailer_len = trailer_len;
1486                 header_len -= xdst->u.dst.xfrm->props.header_len;
1487                 trailer_len -= xdst->u.dst.xfrm->props.trailer_len;
1488         }
1489
1490 out:
1491         return dst0;
1492
1493 put_states:
1494         for (; i < nx; i++)
1495                 xfrm_state_put(xfrm[i]);
1496 free_dst:
1497         if (dst0)
1498                 dst_free(dst0);
1499         dst0 = ERR_PTR(err);
1500         goto out;
1501 }
1502
1503 static int inline
1504 xfrm_dst_alloc_copy(void **target, void *src, int size)
1505 {
1506         if (!*target) {
1507                 *target = kmalloc(size, GFP_ATOMIC);
1508                 if (!*target)
1509                         return -ENOMEM;
1510         }
1511         memcpy(*target, src, size);
1512         return 0;
1513 }
1514
1515 static int inline
1516 xfrm_dst_update_parent(struct dst_entry *dst, struct xfrm_selector *sel)
1517 {
1518 #ifdef CONFIG_XFRM_SUB_POLICY
1519         struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1520         return xfrm_dst_alloc_copy((void **)&(xdst->partner),
1521                                    sel, sizeof(*sel));
1522 #else
1523         return 0;
1524 #endif
1525 }
1526
1527 static int inline
1528 xfrm_dst_update_origin(struct dst_entry *dst, struct flowi *fl)
1529 {
1530 #ifdef CONFIG_XFRM_SUB_POLICY
1531         struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
1532         return xfrm_dst_alloc_copy((void **)&(xdst->origin), fl, sizeof(*fl));
1533 #else
1534         return 0;
1535 #endif
1536 }
1537
1538 static int stale_bundle(struct dst_entry *dst);
1539
1540 /* Main function: finds/creates a bundle for given flow.
1541  *
1542  * At the moment we eat a raw IP route. Mostly to speed up lookups
1543  * on interfaces with disabled IPsec.
1544  */
1545 int __xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
1546                   struct sock *sk, int flags)
1547 {
1548         struct xfrm_policy *policy;
1549         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
1550         int npols;
1551         int pol_dead;
1552         int xfrm_nr;
1553         int pi;
1554         struct xfrm_state *xfrm[XFRM_MAX_DEPTH];
1555         struct dst_entry *dst, *dst_orig = *dst_p;
1556         int nx = 0;
1557         int err;
1558         u32 genid;
1559         u16 family;
1560         u8 dir = policy_to_flow_dir(XFRM_POLICY_OUT);
1561
1562 restart:
1563         genid = atomic_read(&flow_cache_genid);
1564         policy = NULL;
1565         for (pi = 0; pi < ARRAY_SIZE(pols); pi++)
1566                 pols[pi] = NULL;
1567         npols = 0;
1568         pol_dead = 0;
1569         xfrm_nr = 0;
1570
1571         if (sk && sk->sk_policy[XFRM_POLICY_OUT]) {
1572                 policy = xfrm_sk_policy_lookup(sk, XFRM_POLICY_OUT, fl);
1573                 err = PTR_ERR(policy);
1574                 if (IS_ERR(policy)) {
1575                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTPOLERROR);
1576                         goto dropdst;
1577                 }
1578         }
1579
1580         if (!policy) {
1581                 /* To accelerate a bit...  */
1582                 if ((dst_orig->flags & DST_NOXFRM) ||
1583                     !xfrm_policy_count[XFRM_POLICY_OUT])
1584                         goto nopol;
1585
1586                 policy = flow_cache_lookup(fl, dst_orig->ops->family,
1587                                            dir, xfrm_policy_lookup);
1588                 err = PTR_ERR(policy);
1589                 if (IS_ERR(policy)) {
1590                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTPOLERROR);
1591                         goto dropdst;
1592                 }
1593         }
1594
1595         if (!policy)
1596                 goto nopol;
1597
1598         family = dst_orig->ops->family;
1599         pols[0] = policy;
1600         npols ++;
1601         xfrm_nr += pols[0]->xfrm_nr;
1602
1603         err = -ENOENT;
1604         if ((flags & XFRM_LOOKUP_ICMP) && !(policy->flags & XFRM_POLICY_ICMP))
1605                 goto error;
1606
1607         policy->curlft.use_time = get_seconds();
1608
1609         switch (policy->action) {
1610         default:
1611         case XFRM_POLICY_BLOCK:
1612                 /* Prohibit the flow */
1613                 XFRM_INC_STATS(LINUX_MIB_XFRMOUTPOLBLOCK);
1614                 err = -EPERM;
1615                 goto error;
1616
1617         case XFRM_POLICY_ALLOW:
1618 #ifndef CONFIG_XFRM_SUB_POLICY
1619                 if (policy->xfrm_nr == 0) {
1620                         /* Flow passes not transformed. */
1621                         xfrm_pol_put(policy);
1622                         return 0;
1623                 }
1624 #endif
1625
1626                 /* Try to find matching bundle.
1627                  *
1628                  * LATER: help from flow cache. It is optional, this
1629                  * is required only for output policy.
1630                  */
1631                 dst = xfrm_find_bundle(fl, policy, family);
1632                 if (IS_ERR(dst)) {
1633                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1634                         err = PTR_ERR(dst);
1635                         goto error;
1636                 }
1637
1638                 if (dst)
1639                         break;
1640
1641 #ifdef CONFIG_XFRM_SUB_POLICY
1642                 if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1643                         pols[1] = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN,
1644                                                             fl, family,
1645                                                             XFRM_POLICY_OUT);
1646                         if (pols[1]) {
1647                                 if (IS_ERR(pols[1])) {
1648                                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTPOLERROR);
1649                                         err = PTR_ERR(pols[1]);
1650                                         goto error;
1651                                 }
1652                                 if (pols[1]->action == XFRM_POLICY_BLOCK) {
1653                                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTPOLBLOCK);
1654                                         err = -EPERM;
1655                                         goto error;
1656                                 }
1657                                 npols ++;
1658                                 xfrm_nr += pols[1]->xfrm_nr;
1659                         }
1660                 }
1661
1662                 /*
1663                  * Because neither flowi nor bundle information knows about
1664                  * transformation template size. On more than one policy usage
1665                  * we can realize whether all of them is bypass or not after
1666                  * they are searched. See above not-transformed bypass
1667                  * is surrounded by non-sub policy configuration, too.
1668                  */
1669                 if (xfrm_nr == 0) {
1670                         /* Flow passes not transformed. */
1671                         xfrm_pols_put(pols, npols);
1672                         return 0;
1673                 }
1674
1675 #endif
1676                 nx = xfrm_tmpl_resolve(pols, npols, fl, xfrm, family);
1677
1678                 if (unlikely(nx<0)) {
1679                         err = nx;
1680                         if (err == -EAGAIN && sysctl_xfrm_larval_drop) {
1681                                 /* EREMOTE tells the caller to generate
1682                                  * a one-shot blackhole route.
1683                                  */
1684                                 XFRM_INC_STATS(LINUX_MIB_XFRMOUTNOSTATES);
1685                                 xfrm_pol_put(policy);
1686                                 return -EREMOTE;
1687                         }
1688                         if (err == -EAGAIN && (flags & XFRM_LOOKUP_WAIT)) {
1689                                 DECLARE_WAITQUEUE(wait, current);
1690
1691                                 add_wait_queue(&init_net.xfrm.km_waitq, &wait);
1692                                 set_current_state(TASK_INTERRUPTIBLE);
1693                                 schedule();
1694                                 set_current_state(TASK_RUNNING);
1695                                 remove_wait_queue(&init_net.xfrm.km_waitq, &wait);
1696
1697                                 nx = xfrm_tmpl_resolve(pols, npols, fl, xfrm, family);
1698
1699                                 if (nx == -EAGAIN && signal_pending(current)) {
1700                                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTNOSTATES);
1701                                         err = -ERESTART;
1702                                         goto error;
1703                                 }
1704                                 if (nx == -EAGAIN ||
1705                                     genid != atomic_read(&flow_cache_genid)) {
1706                                         xfrm_pols_put(pols, npols);
1707                                         goto restart;
1708                                 }
1709                                 err = nx;
1710                         }
1711                         if (err < 0) {
1712                                 XFRM_INC_STATS(LINUX_MIB_XFRMOUTNOSTATES);
1713                                 goto error;
1714                         }
1715                 }
1716                 if (nx == 0) {
1717                         /* Flow passes not transformed. */
1718                         xfrm_pols_put(pols, npols);
1719                         return 0;
1720                 }
1721
1722                 dst = xfrm_bundle_create(policy, xfrm, nx, fl, dst_orig);
1723                 err = PTR_ERR(dst);
1724                 if (IS_ERR(dst)) {
1725                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTBUNDLEGENERROR);
1726                         goto error;
1727                 }
1728
1729                 for (pi = 0; pi < npols; pi++) {
1730                         read_lock_bh(&pols[pi]->lock);
1731                         pol_dead |= pols[pi]->walk.dead;
1732                         read_unlock_bh(&pols[pi]->lock);
1733                 }
1734
1735                 write_lock_bh(&policy->lock);
1736                 if (unlikely(pol_dead || stale_bundle(dst))) {
1737                         /* Wow! While we worked on resolving, this
1738                          * policy has gone. Retry. It is not paranoia,
1739                          * we just cannot enlist new bundle to dead object.
1740                          * We can't enlist stable bundles either.
1741                          */
1742                         write_unlock_bh(&policy->lock);
1743                         dst_free(dst);
1744
1745                         if (pol_dead)
1746                                 XFRM_INC_STATS(LINUX_MIB_XFRMOUTPOLDEAD);
1747                         else
1748                                 XFRM_INC_STATS(LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1749                         err = -EHOSTUNREACH;
1750                         goto error;
1751                 }
1752
1753                 if (npols > 1)
1754                         err = xfrm_dst_update_parent(dst, &pols[1]->selector);
1755                 else
1756                         err = xfrm_dst_update_origin(dst, fl);
1757                 if (unlikely(err)) {
1758                         write_unlock_bh(&policy->lock);
1759                         dst_free(dst);
1760                         XFRM_INC_STATS(LINUX_MIB_XFRMOUTBUNDLECHECKERROR);
1761                         goto error;
1762                 }
1763
1764                 dst->next = policy->bundles;
1765                 policy->bundles = dst;
1766                 dst_hold(dst);
1767                 write_unlock_bh(&policy->lock);
1768         }
1769         *dst_p = dst;
1770         dst_release(dst_orig);
1771         xfrm_pols_put(pols, npols);
1772         return 0;
1773
1774 error:
1775         xfrm_pols_put(pols, npols);
1776 dropdst:
1777         dst_release(dst_orig);
1778         *dst_p = NULL;
1779         return err;
1780
1781 nopol:
1782         err = -ENOENT;
1783         if (flags & XFRM_LOOKUP_ICMP)
1784                 goto dropdst;
1785         return 0;
1786 }
1787 EXPORT_SYMBOL(__xfrm_lookup);
1788
1789 int xfrm_lookup(struct dst_entry **dst_p, struct flowi *fl,
1790                 struct sock *sk, int flags)
1791 {
1792         int err = __xfrm_lookup(dst_p, fl, sk, flags);
1793
1794         if (err == -EREMOTE) {
1795                 dst_release(*dst_p);
1796                 *dst_p = NULL;
1797                 err = -EAGAIN;
1798         }
1799
1800         return err;
1801 }
1802 EXPORT_SYMBOL(xfrm_lookup);
1803
1804 static inline int
1805 xfrm_secpath_reject(int idx, struct sk_buff *skb, struct flowi *fl)
1806 {
1807         struct xfrm_state *x;
1808
1809         if (!skb->sp || idx < 0 || idx >= skb->sp->len)
1810                 return 0;
1811         x = skb->sp->xvec[idx];
1812         if (!x->type->reject)
1813                 return 0;
1814         return x->type->reject(x, skb, fl);
1815 }
1816
1817 /* When skb is transformed back to its "native" form, we have to
1818  * check policy restrictions. At the moment we make this in maximally
1819  * stupid way. Shame on me. :-) Of course, connected sockets must
1820  * have policy cached at them.
1821  */
1822
1823 static inline int
1824 xfrm_state_ok(struct xfrm_tmpl *tmpl, struct xfrm_state *x,
1825               unsigned short family)
1826 {
1827         if (xfrm_state_kern(x))
1828                 return tmpl->optional && !xfrm_state_addr_cmp(tmpl, x, tmpl->encap_family);
1829         return  x->id.proto == tmpl->id.proto &&
1830                 (x->id.spi == tmpl->id.spi || !tmpl->id.spi) &&
1831                 (x->props.reqid == tmpl->reqid || !tmpl->reqid) &&
1832                 x->props.mode == tmpl->mode &&
1833                 (tmpl->allalgs || (tmpl->aalgos & (1<<x->props.aalgo)) ||
1834                  !(xfrm_id_proto_match(tmpl->id.proto, IPSEC_PROTO_ANY))) &&
1835                 !(x->props.mode != XFRM_MODE_TRANSPORT &&
1836                   xfrm_state_addr_cmp(tmpl, x, family));
1837 }
1838
1839 /*
1840  * 0 or more than 0 is returned when validation is succeeded (either bypass
1841  * because of optional transport mode, or next index of the mathced secpath
1842  * state with the template.
1843  * -1 is returned when no matching template is found.
1844  * Otherwise "-2 - errored_index" is returned.
1845  */
1846 static inline int
1847 xfrm_policy_ok(struct xfrm_tmpl *tmpl, struct sec_path *sp, int start,
1848                unsigned short family)
1849 {
1850         int idx = start;
1851
1852         if (tmpl->optional) {
1853                 if (tmpl->mode == XFRM_MODE_TRANSPORT)
1854                         return start;
1855         } else
1856                 start = -1;
1857         for (; idx < sp->len; idx++) {
1858                 if (xfrm_state_ok(tmpl, sp->xvec[idx], family))
1859                         return ++idx;
1860                 if (sp->xvec[idx]->props.mode != XFRM_MODE_TRANSPORT) {
1861                         if (start == -1)
1862                                 start = -2-idx;
1863                         break;
1864                 }
1865         }
1866         return start;
1867 }
1868
1869 int __xfrm_decode_session(struct sk_buff *skb, struct flowi *fl,
1870                           unsigned int family, int reverse)
1871 {
1872         struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
1873         int err;
1874
1875         if (unlikely(afinfo == NULL))
1876                 return -EAFNOSUPPORT;
1877
1878         afinfo->decode_session(skb, fl, reverse);
1879         err = security_xfrm_decode_session(skb, &fl->secid);
1880         xfrm_policy_put_afinfo(afinfo);
1881         return err;
1882 }
1883 EXPORT_SYMBOL(__xfrm_decode_session);
1884
1885 static inline int secpath_has_nontransport(struct sec_path *sp, int k, int *idxp)
1886 {
1887         for (; k < sp->len; k++) {
1888                 if (sp->xvec[k]->props.mode != XFRM_MODE_TRANSPORT) {
1889                         *idxp = k;
1890                         return 1;
1891                 }
1892         }
1893
1894         return 0;
1895 }
1896
1897 int __xfrm_policy_check(struct sock *sk, int dir, struct sk_buff *skb,
1898                         unsigned short family)
1899 {
1900         struct xfrm_policy *pol;
1901         struct xfrm_policy *pols[XFRM_POLICY_TYPE_MAX];
1902         int npols = 0;
1903         int xfrm_nr;
1904         int pi;
1905         int reverse;
1906         struct flowi fl;
1907         u8 fl_dir;
1908         int xerr_idx = -1;
1909
1910         reverse = dir & ~XFRM_POLICY_MASK;
1911         dir &= XFRM_POLICY_MASK;
1912         fl_dir = policy_to_flow_dir(dir);
1913
1914         if (__xfrm_decode_session(skb, &fl, family, reverse) < 0) {
1915                 XFRM_INC_STATS(LINUX_MIB_XFRMINHDRERROR);
1916                 return 0;
1917         }
1918
1919         nf_nat_decode_session(skb, &fl, family);
1920
1921         /* First, check used SA against their selectors. */
1922         if (skb->sp) {
1923                 int i;
1924
1925                 for (i=skb->sp->len-1; i>=0; i--) {
1926                         struct xfrm_state *x = skb->sp->xvec[i];
1927                         if (!xfrm_selector_match(&x->sel, &fl, family)) {
1928                                 XFRM_INC_STATS(LINUX_MIB_XFRMINSTATEMISMATCH);
1929                                 return 0;
1930                         }
1931                 }
1932         }
1933
1934         pol = NULL;
1935         if (sk && sk->sk_policy[dir]) {
1936                 pol = xfrm_sk_policy_lookup(sk, dir, &fl);
1937                 if (IS_ERR(pol)) {
1938                         XFRM_INC_STATS(LINUX_MIB_XFRMINPOLERROR);
1939                         return 0;
1940                 }
1941         }
1942
1943         if (!pol)
1944                 pol = flow_cache_lookup(&fl, family, fl_dir,
1945                                         xfrm_policy_lookup);
1946
1947         if (IS_ERR(pol)) {
1948                 XFRM_INC_STATS(LINUX_MIB_XFRMINPOLERROR);
1949                 return 0;
1950         }
1951
1952         if (!pol) {
1953                 if (skb->sp && secpath_has_nontransport(skb->sp, 0, &xerr_idx)) {
1954                         xfrm_secpath_reject(xerr_idx, skb, &fl);
1955                         XFRM_INC_STATS(LINUX_MIB_XFRMINNOPOLS);
1956                         return 0;
1957                 }
1958                 return 1;
1959         }
1960
1961         pol->curlft.use_time = get_seconds();
1962
1963         pols[0] = pol;
1964         npols ++;
1965 #ifdef CONFIG_XFRM_SUB_POLICY
1966         if (pols[0]->type != XFRM_POLICY_TYPE_MAIN) {
1967                 pols[1] = xfrm_policy_lookup_bytype(XFRM_POLICY_TYPE_MAIN,
1968                                                     &fl, family,
1969                                                     XFRM_POLICY_IN);
1970                 if (pols[1]) {
1971                         if (IS_ERR(pols[1])) {
1972                                 XFRM_INC_STATS(LINUX_MIB_XFRMINPOLERROR);
1973                                 return 0;
1974                         }
1975                         pols[1]->curlft.use_time = get_seconds();
1976                         npols ++;
1977                 }
1978         }
1979 #endif
1980
1981         if (pol->action == XFRM_POLICY_ALLOW) {
1982                 struct sec_path *sp;
1983                 static struct sec_path dummy;
1984                 struct xfrm_tmpl *tp[XFRM_MAX_DEPTH];
1985                 struct xfrm_tmpl *stp[XFRM_MAX_DEPTH];
1986                 struct xfrm_tmpl **tpp = tp;
1987                 int ti = 0;
1988                 int i, k;
1989
1990                 if ((sp = skb->sp) == NULL)
1991                         sp = &dummy;
1992
1993                 for (pi = 0; pi < npols; pi++) {
1994                         if (pols[pi] != pol &&
1995                             pols[pi]->action != XFRM_POLICY_ALLOW) {
1996                                 XFRM_INC_STATS(LINUX_MIB_XFRMINPOLBLOCK);
1997                                 goto reject;
1998                         }
1999                         if (ti + pols[pi]->xfrm_nr >= XFRM_MAX_DEPTH) {
2000                                 XFRM_INC_STATS(LINUX_MIB_XFRMINBUFFERERROR);
2001                                 goto reject_error;
2002                         }
2003                         for (i = 0; i < pols[pi]->xfrm_nr; i++)
2004                                 tpp[ti++] = &pols[pi]->xfrm_vec[i];
2005                 }
2006                 xfrm_nr = ti;
2007                 if (npols > 1) {
2008                         xfrm_tmpl_sort(stp, tpp, xfrm_nr, family);
2009                         tpp = stp;
2010                 }
2011
2012                 /* For each tunnel xfrm, find the first matching tmpl.
2013                  * For each tmpl before that, find corresponding xfrm.
2014                  * Order is _important_. Later we will implement
2015                  * some barriers, but at the moment barriers
2016                  * are implied between each two transformations.
2017                  */
2018                 for (i = xfrm_nr-1, k = 0; i >= 0; i--) {
2019                         k = xfrm_policy_ok(tpp[i], sp, k, family);
2020                         if (k < 0) {
2021                                 if (k < -1)
2022                                         /* "-2 - errored_index" returned */
2023                                         xerr_idx = -(2+k);
2024                                 XFRM_INC_STATS(LINUX_MIB_XFRMINTMPLMISMATCH);
2025                                 goto reject;
2026                         }
2027                 }
2028
2029                 if (secpath_has_nontransport(sp, k, &xerr_idx)) {
2030                         XFRM_INC_STATS(LINUX_MIB_XFRMINTMPLMISMATCH);
2031                         goto reject;
2032                 }
2033
2034                 xfrm_pols_put(pols, npols);
2035                 return 1;
2036         }
2037         XFRM_INC_STATS(LINUX_MIB_XFRMINPOLBLOCK);
2038
2039 reject:
2040         xfrm_secpath_reject(xerr_idx, skb, &fl);
2041 reject_error:
2042         xfrm_pols_put(pols, npols);
2043         return 0;
2044 }
2045 EXPORT_SYMBOL(__xfrm_policy_check);
2046
2047 int __xfrm_route_forward(struct sk_buff *skb, unsigned short family)
2048 {
2049         struct flowi fl;
2050
2051         if (xfrm_decode_session(skb, &fl, family) < 0) {
2052                 /* XXX: we should have something like FWDHDRERROR here. */
2053                 XFRM_INC_STATS(LINUX_MIB_XFRMINHDRERROR);
2054                 return 0;
2055         }
2056
2057         return xfrm_lookup(&skb->dst, &fl, NULL, 0) == 0;
2058 }
2059 EXPORT_SYMBOL(__xfrm_route_forward);
2060
2061 /* Optimize later using cookies and generation ids. */
2062
2063 static struct dst_entry *xfrm_dst_check(struct dst_entry *dst, u32 cookie)
2064 {
2065         /* Code (such as __xfrm4_bundle_create()) sets dst->obsolete
2066          * to "-1" to force all XFRM destinations to get validated by
2067          * dst_ops->check on every use.  We do this because when a
2068          * normal route referenced by an XFRM dst is obsoleted we do
2069          * not go looking around for all parent referencing XFRM dsts
2070          * so that we can invalidate them.  It is just too much work.
2071          * Instead we make the checks here on every use.  For example:
2072          *
2073          *      XFRM dst A --> IPv4 dst X
2074          *
2075          * X is the "xdst->route" of A (X is also the "dst->path" of A
2076          * in this example).  If X is marked obsolete, "A" will not
2077          * notice.  That's what we are validating here via the
2078          * stale_bundle() check.
2079          *
2080          * When a policy's bundle is pruned, we dst_free() the XFRM
2081          * dst which causes it's ->obsolete field to be set to a
2082          * positive non-zero integer.  If an XFRM dst has been pruned
2083          * like this, we want to force a new route lookup.
2084          */
2085         if (dst->obsolete < 0 && !stale_bundle(dst))
2086                 return dst;
2087
2088         return NULL;
2089 }
2090
2091 static int stale_bundle(struct dst_entry *dst)
2092 {
2093         return !xfrm_bundle_ok(NULL, (struct xfrm_dst *)dst, NULL, AF_UNSPEC, 0);
2094 }
2095
2096 void xfrm_dst_ifdown(struct dst_entry *dst, struct net_device *dev)
2097 {
2098         while ((dst = dst->child) && dst->xfrm && dst->dev == dev) {
2099                 dst->dev = dev_net(dev)->loopback_dev;
2100                 dev_hold(dst->dev);
2101                 dev_put(dev);
2102         }
2103 }
2104 EXPORT_SYMBOL(xfrm_dst_ifdown);
2105
2106 static void xfrm_link_failure(struct sk_buff *skb)
2107 {
2108         /* Impossible. Such dst must be popped before reaches point of failure. */
2109         return;
2110 }
2111
2112 static struct dst_entry *xfrm_negative_advice(struct dst_entry *dst)
2113 {
2114         if (dst) {
2115                 if (dst->obsolete) {
2116                         dst_release(dst);
2117                         dst = NULL;
2118                 }
2119         }
2120         return dst;
2121 }
2122
2123 static void prune_one_bundle(struct xfrm_policy *pol, int (*func)(struct dst_entry *), struct dst_entry **gc_list_p)
2124 {
2125         struct dst_entry *dst, **dstp;
2126
2127         write_lock(&pol->lock);
2128         dstp = &pol->bundles;
2129         while ((dst=*dstp) != NULL) {
2130                 if (func(dst)) {
2131                         *dstp = dst->next;
2132                         dst->next = *gc_list_p;
2133                         *gc_list_p = dst;
2134                 } else {
2135                         dstp = &dst->next;
2136                 }
2137         }
2138         write_unlock(&pol->lock);
2139 }
2140
2141 static void xfrm_prune_bundles(int (*func)(struct dst_entry *))
2142 {
2143         struct dst_entry *gc_list = NULL;
2144         int dir;
2145
2146         read_lock_bh(&xfrm_policy_lock);
2147         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2148                 struct xfrm_policy *pol;
2149                 struct hlist_node *entry;
2150                 struct hlist_head *table;
2151                 int i;
2152
2153                 hlist_for_each_entry(pol, entry,
2154                                      &init_net.xfrm.policy_inexact[dir], bydst)
2155                         prune_one_bundle(pol, func, &gc_list);
2156
2157                 table = xfrm_policy_bydst[dir].table;
2158                 for (i = xfrm_policy_bydst[dir].hmask; i >= 0; i--) {
2159                         hlist_for_each_entry(pol, entry, table + i, bydst)
2160                                 prune_one_bundle(pol, func, &gc_list);
2161                 }
2162         }
2163         read_unlock_bh(&xfrm_policy_lock);
2164
2165         while (gc_list) {
2166                 struct dst_entry *dst = gc_list;
2167                 gc_list = dst->next;
2168                 dst_free(dst);
2169         }
2170 }
2171
2172 static int unused_bundle(struct dst_entry *dst)
2173 {
2174         return !atomic_read(&dst->__refcnt);
2175 }
2176
2177 static void __xfrm_garbage_collect(void)
2178 {
2179         xfrm_prune_bundles(unused_bundle);
2180 }
2181
2182 static int xfrm_flush_bundles(void)
2183 {
2184         xfrm_prune_bundles(stale_bundle);
2185         return 0;
2186 }
2187
2188 static void xfrm_init_pmtu(struct dst_entry *dst)
2189 {
2190         do {
2191                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2192                 u32 pmtu, route_mtu_cached;
2193
2194                 pmtu = dst_mtu(dst->child);
2195                 xdst->child_mtu_cached = pmtu;
2196
2197                 pmtu = xfrm_state_mtu(dst->xfrm, pmtu);
2198
2199                 route_mtu_cached = dst_mtu(xdst->route);
2200                 xdst->route_mtu_cached = route_mtu_cached;
2201
2202                 if (pmtu > route_mtu_cached)
2203                         pmtu = route_mtu_cached;
2204
2205                 dst->metrics[RTAX_MTU-1] = pmtu;
2206         } while ((dst = dst->next));
2207 }
2208
2209 /* Check that the bundle accepts the flow and its components are
2210  * still valid.
2211  */
2212
2213 int xfrm_bundle_ok(struct xfrm_policy *pol, struct xfrm_dst *first,
2214                 struct flowi *fl, int family, int strict)
2215 {
2216         struct dst_entry *dst = &first->u.dst;
2217         struct xfrm_dst *last;
2218         u32 mtu;
2219
2220         if (!dst_check(dst->path, ((struct xfrm_dst *)dst)->path_cookie) ||
2221             (dst->dev && !netif_running(dst->dev)))
2222                 return 0;
2223 #ifdef CONFIG_XFRM_SUB_POLICY
2224         if (fl) {
2225                 if (first->origin && !flow_cache_uli_match(first->origin, fl))
2226                         return 0;
2227                 if (first->partner &&
2228                     !xfrm_selector_match(first->partner, fl, family))
2229                         return 0;
2230         }
2231 #endif
2232
2233         last = NULL;
2234
2235         do {
2236                 struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
2237
2238                 if (fl && !xfrm_selector_match(&dst->xfrm->sel, fl, family))
2239                         return 0;
2240                 if (fl && pol &&
2241                     !security_xfrm_state_pol_flow_match(dst->xfrm, pol, fl))
2242                         return 0;
2243                 if (dst->xfrm->km.state != XFRM_STATE_VALID)
2244                         return 0;
2245                 if (xdst->genid != dst->xfrm->genid)
2246                         return 0;
2247
2248                 if (strict && fl &&
2249                     !(dst->xfrm->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) &&
2250                     !xfrm_state_addr_flow_check(dst->xfrm, fl, family))
2251                         return 0;
2252
2253                 mtu = dst_mtu(dst->child);
2254                 if (xdst->child_mtu_cached != mtu) {
2255                         last = xdst;
2256                         xdst->child_mtu_cached = mtu;
2257                 }
2258
2259                 if (!dst_check(xdst->route, xdst->route_cookie))
2260                         return 0;
2261                 mtu = dst_mtu(xdst->route);
2262                 if (xdst->route_mtu_cached != mtu) {
2263                         last = xdst;
2264                         xdst->route_mtu_cached = mtu;
2265                 }
2266
2267                 dst = dst->child;
2268         } while (dst->xfrm);
2269
2270         if (likely(!last))
2271                 return 1;
2272
2273         mtu = last->child_mtu_cached;
2274         for (;;) {
2275                 dst = &last->u.dst;
2276
2277                 mtu = xfrm_state_mtu(dst->xfrm, mtu);
2278                 if (mtu > last->route_mtu_cached)
2279                         mtu = last->route_mtu_cached;
2280                 dst->metrics[RTAX_MTU-1] = mtu;
2281
2282                 if (last == first)
2283                         break;
2284
2285                 last = (struct xfrm_dst *)last->u.dst.next;
2286                 last->child_mtu_cached = mtu;
2287         }
2288
2289         return 1;
2290 }
2291
2292 EXPORT_SYMBOL(xfrm_bundle_ok);
2293
2294 int xfrm_policy_register_afinfo(struct xfrm_policy_afinfo *afinfo)
2295 {
2296         int err = 0;
2297         if (unlikely(afinfo == NULL))
2298                 return -EINVAL;
2299         if (unlikely(afinfo->family >= NPROTO))
2300                 return -EAFNOSUPPORT;
2301         write_lock_bh(&xfrm_policy_afinfo_lock);
2302         if (unlikely(xfrm_policy_afinfo[afinfo->family] != NULL))
2303                 err = -ENOBUFS;
2304         else {
2305                 struct dst_ops *dst_ops = afinfo->dst_ops;
2306                 if (likely(dst_ops->kmem_cachep == NULL))
2307                         dst_ops->kmem_cachep = xfrm_dst_cache;
2308                 if (likely(dst_ops->check == NULL))
2309                         dst_ops->check = xfrm_dst_check;
2310                 if (likely(dst_ops->negative_advice == NULL))
2311                         dst_ops->negative_advice = xfrm_negative_advice;
2312                 if (likely(dst_ops->link_failure == NULL))
2313                         dst_ops->link_failure = xfrm_link_failure;
2314                 if (likely(afinfo->garbage_collect == NULL))
2315                         afinfo->garbage_collect = __xfrm_garbage_collect;
2316                 xfrm_policy_afinfo[afinfo->family] = afinfo;
2317         }
2318         write_unlock_bh(&xfrm_policy_afinfo_lock);
2319         return err;
2320 }
2321 EXPORT_SYMBOL(xfrm_policy_register_afinfo);
2322
2323 int xfrm_policy_unregister_afinfo(struct xfrm_policy_afinfo *afinfo)
2324 {
2325         int err = 0;
2326         if (unlikely(afinfo == NULL))
2327                 return -EINVAL;
2328         if (unlikely(afinfo->family >= NPROTO))
2329                 return -EAFNOSUPPORT;
2330         write_lock_bh(&xfrm_policy_afinfo_lock);
2331         if (likely(xfrm_policy_afinfo[afinfo->family] != NULL)) {
2332                 if (unlikely(xfrm_policy_afinfo[afinfo->family] != afinfo))
2333                         err = -EINVAL;
2334                 else {
2335                         struct dst_ops *dst_ops = afinfo->dst_ops;
2336                         xfrm_policy_afinfo[afinfo->family] = NULL;
2337                         dst_ops->kmem_cachep = NULL;
2338                         dst_ops->check = NULL;
2339                         dst_ops->negative_advice = NULL;
2340                         dst_ops->link_failure = NULL;
2341                         afinfo->garbage_collect = NULL;
2342                 }
2343         }
2344         write_unlock_bh(&xfrm_policy_afinfo_lock);
2345         return err;
2346 }
2347 EXPORT_SYMBOL(xfrm_policy_unregister_afinfo);
2348
2349 static struct xfrm_policy_afinfo *xfrm_policy_get_afinfo(unsigned short family)
2350 {
2351         struct xfrm_policy_afinfo *afinfo;
2352         if (unlikely(family >= NPROTO))
2353                 return NULL;
2354         read_lock(&xfrm_policy_afinfo_lock);
2355         afinfo = xfrm_policy_afinfo[family];
2356         if (unlikely(!afinfo))
2357                 read_unlock(&xfrm_policy_afinfo_lock);
2358         return afinfo;
2359 }
2360
2361 static void xfrm_policy_put_afinfo(struct xfrm_policy_afinfo *afinfo)
2362 {
2363         read_unlock(&xfrm_policy_afinfo_lock);
2364 }
2365
2366 static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
2367 {
2368         struct net_device *dev = ptr;
2369
2370         if (!net_eq(dev_net(dev), &init_net))
2371                 return NOTIFY_DONE;
2372
2373         switch (event) {
2374         case NETDEV_DOWN:
2375                 xfrm_flush_bundles();
2376         }
2377         return NOTIFY_DONE;
2378 }
2379
2380 static struct notifier_block xfrm_dev_notifier = {
2381         .notifier_call  = xfrm_dev_event,
2382 };
2383
2384 #ifdef CONFIG_XFRM_STATISTICS
2385 static int __init xfrm_statistics_init(void)
2386 {
2387         if (snmp_mib_init((void **)xfrm_statistics,
2388                           sizeof(struct linux_xfrm_mib)) < 0)
2389                 return -ENOMEM;
2390         return 0;
2391 }
2392 #endif
2393
2394 static int __net_init xfrm_policy_init(struct net *net)
2395 {
2396         unsigned int hmask, sz;
2397         int dir;
2398
2399         if (net_eq(net, &init_net))
2400                 xfrm_dst_cache = kmem_cache_create("xfrm_dst_cache",
2401                                            sizeof(struct xfrm_dst),
2402                                            0, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
2403                                            NULL);
2404
2405         hmask = 8 - 1;
2406         sz = (hmask+1) * sizeof(struct hlist_head);
2407
2408         net->xfrm.policy_byidx = xfrm_hash_alloc(sz);
2409         if (!net->xfrm.policy_byidx)
2410                 goto out_byidx;
2411         net->xfrm.policy_idx_hmask = hmask;
2412
2413         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2414                 struct xfrm_policy_hash *htab;
2415
2416                 INIT_HLIST_HEAD(&net->xfrm.policy_inexact[dir]);
2417
2418                 htab = &xfrm_policy_bydst[dir];
2419                 htab->table = xfrm_hash_alloc(sz);
2420                 htab->hmask = hmask;
2421                 if (!htab->table)
2422                         panic("XFRM: failed to allocate bydst hash\n");
2423         }
2424
2425         INIT_LIST_HEAD(&net->xfrm.policy_all);
2426         if (net_eq(net, &init_net))
2427                 register_netdevice_notifier(&xfrm_dev_notifier);
2428         return 0;
2429
2430 out_byidx:
2431         return -ENOMEM;
2432 }
2433
2434 static void xfrm_policy_fini(struct net *net)
2435 {
2436         unsigned int sz;
2437         int dir;
2438
2439         WARN_ON(!list_empty(&net->xfrm.policy_all));
2440
2441         for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2442                 WARN_ON(!hlist_empty(&net->xfrm.policy_inexact[dir]));
2443         }
2444
2445         sz = (net->xfrm.policy_idx_hmask + 1) * sizeof(struct hlist_head);
2446         WARN_ON(!hlist_empty(net->xfrm.policy_byidx));
2447         xfrm_hash_free(net->xfrm.policy_byidx, sz);
2448 }
2449
2450 static int __net_init xfrm_net_init(struct net *net)
2451 {
2452         int rv;
2453
2454         rv = xfrm_state_init(net);
2455         if (rv < 0)
2456                 goto out_state;
2457         rv = xfrm_policy_init(net);
2458         if (rv < 0)
2459                 goto out_policy;
2460         return 0;
2461
2462 out_policy:
2463         xfrm_state_fini(net);
2464 out_state:
2465         return rv;
2466 }
2467
2468 static void __net_exit xfrm_net_exit(struct net *net)
2469 {
2470         xfrm_policy_fini(net);
2471         xfrm_state_fini(net);
2472 }
2473
2474 static struct pernet_operations __net_initdata xfrm_net_ops = {
2475         .init = xfrm_net_init,
2476         .exit = xfrm_net_exit,
2477 };
2478
2479 void __init xfrm_init(void)
2480 {
2481         register_pernet_subsys(&xfrm_net_ops);
2482 #ifdef CONFIG_XFRM_STATISTICS
2483         xfrm_statistics_init();
2484 #endif
2485         xfrm_input_init();
2486 #ifdef CONFIG_XFRM_STATISTICS
2487         xfrm_proc_init();
2488 #endif
2489 }
2490
2491 #ifdef CONFIG_AUDITSYSCALL
2492 static void xfrm_audit_common_policyinfo(struct xfrm_policy *xp,
2493                                          struct audit_buffer *audit_buf)
2494 {
2495         struct xfrm_sec_ctx *ctx = xp->security;
2496         struct xfrm_selector *sel = &xp->selector;
2497
2498         if (ctx)
2499                 audit_log_format(audit_buf, " sec_alg=%u sec_doi=%u sec_obj=%s",
2500                                  ctx->ctx_alg, ctx->ctx_doi, ctx->ctx_str);
2501
2502         switch(sel->family) {
2503         case AF_INET:
2504                 audit_log_format(audit_buf, " src=%pI4", &sel->saddr.a4);
2505                 if (sel->prefixlen_s != 32)
2506                         audit_log_format(audit_buf, " src_prefixlen=%d",
2507                                          sel->prefixlen_s);
2508                 audit_log_format(audit_buf, " dst=%pI4", &sel->daddr.a4);
2509                 if (sel->prefixlen_d != 32)
2510                         audit_log_format(audit_buf, " dst_prefixlen=%d",
2511                                          sel->prefixlen_d);
2512                 break;
2513         case AF_INET6:
2514                 audit_log_format(audit_buf, " src=%pI6", sel->saddr.a6);
2515                 if (sel->prefixlen_s != 128)
2516                         audit_log_format(audit_buf, " src_prefixlen=%d",
2517                                          sel->prefixlen_s);
2518                 audit_log_format(audit_buf, " dst=%pI6", sel->daddr.a6);
2519                 if (sel->prefixlen_d != 128)
2520                         audit_log_format(audit_buf, " dst_prefixlen=%d",
2521                                          sel->prefixlen_d);
2522                 break;
2523         }
2524 }
2525
2526 void xfrm_audit_policy_add(struct xfrm_policy *xp, int result,
2527                            uid_t auid, u32 sessionid, u32 secid)
2528 {
2529         struct audit_buffer *audit_buf;
2530
2531         audit_buf = xfrm_audit_start("SPD-add");
2532         if (audit_buf == NULL)
2533                 return;
2534         xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
2535         audit_log_format(audit_buf, " res=%u", result);
2536         xfrm_audit_common_policyinfo(xp, audit_buf);
2537         audit_log_end(audit_buf);
2538 }
2539 EXPORT_SYMBOL_GPL(xfrm_audit_policy_add);
2540
2541 void xfrm_audit_policy_delete(struct xfrm_policy *xp, int result,
2542                               uid_t auid, u32 sessionid, u32 secid)
2543 {
2544         struct audit_buffer *audit_buf;
2545
2546         audit_buf = xfrm_audit_start("SPD-delete");
2547         if (audit_buf == NULL)
2548                 return;
2549         xfrm_audit_helper_usrinfo(auid, sessionid, secid, audit_buf);
2550         audit_log_format(audit_buf, " res=%u", result);
2551         xfrm_audit_common_policyinfo(xp, audit_buf);
2552         audit_log_end(audit_buf);
2553 }
2554 EXPORT_SYMBOL_GPL(xfrm_audit_policy_delete);
2555 #endif
2556
2557 #ifdef CONFIG_XFRM_MIGRATE
2558 static int xfrm_migrate_selector_match(struct xfrm_selector *sel_cmp,
2559                                        struct xfrm_selector *sel_tgt)
2560 {
2561         if (sel_cmp->proto == IPSEC_ULPROTO_ANY) {
2562                 if (sel_tgt->family == sel_cmp->family &&
2563                     xfrm_addr_cmp(&sel_tgt->daddr, &sel_cmp->daddr,
2564                                   sel_cmp->family) == 0 &&
2565                     xfrm_addr_cmp(&sel_tgt->saddr, &sel_cmp->saddr,
2566                                   sel_cmp->family) == 0 &&
2567                     sel_tgt->prefixlen_d == sel_cmp->prefixlen_d &&
2568                     sel_tgt->prefixlen_s == sel_cmp->prefixlen_s) {
2569                         return 1;
2570                 }
2571         } else {
2572                 if (memcmp(sel_tgt, sel_cmp, sizeof(*sel_tgt)) == 0) {
2573                         return 1;
2574                 }
2575         }
2576         return 0;
2577 }
2578
2579 static struct xfrm_policy * xfrm_migrate_policy_find(struct xfrm_selector *sel,
2580                                                      u8 dir, u8 type)
2581 {
2582         struct xfrm_policy *pol, *ret = NULL;
2583         struct hlist_node *entry;
2584         struct hlist_head *chain;
2585         u32 priority = ~0U;
2586
2587         read_lock_bh(&xfrm_policy_lock);
2588         chain = policy_hash_direct(&sel->daddr, &sel->saddr, sel->family, dir);
2589         hlist_for_each_entry(pol, entry, chain, bydst) {
2590                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
2591                     pol->type == type) {
2592                         ret = pol;
2593                         priority = ret->priority;
2594                         break;
2595                 }
2596         }
2597         chain = &init_net.xfrm.policy_inexact[dir];
2598         hlist_for_each_entry(pol, entry, chain, bydst) {
2599                 if (xfrm_migrate_selector_match(sel, &pol->selector) &&
2600                     pol->type == type &&
2601                     pol->priority < priority) {
2602                         ret = pol;
2603                         break;
2604                 }
2605         }
2606
2607         if (ret)
2608                 xfrm_pol_hold(ret);
2609
2610         read_unlock_bh(&xfrm_policy_lock);
2611
2612         return ret;
2613 }
2614
2615 static int migrate_tmpl_match(struct xfrm_migrate *m, struct xfrm_tmpl *t)
2616 {
2617         int match = 0;
2618
2619         if (t->mode == m->mode && t->id.proto == m->proto &&
2620             (m->reqid == 0 || t->reqid == m->reqid)) {
2621                 switch (t->mode) {
2622                 case XFRM_MODE_TUNNEL:
2623                 case XFRM_MODE_BEET:
2624                         if (xfrm_addr_cmp(&t->id.daddr, &m->old_daddr,
2625                                           m->old_family) == 0 &&
2626                             xfrm_addr_cmp(&t->saddr, &m->old_saddr,
2627                                           m->old_family) == 0) {
2628                                 match = 1;
2629                         }
2630                         break;
2631                 case XFRM_MODE_TRANSPORT:
2632                         /* in case of transport mode, template does not store
2633                            any IP addresses, hence we just compare mode and
2634                            protocol */
2635                         match = 1;
2636                         break;
2637                 default:
2638                         break;
2639                 }
2640         }
2641         return match;
2642 }
2643
2644 /* update endpoint address(es) of template(s) */
2645 static int xfrm_policy_migrate(struct xfrm_policy *pol,
2646                                struct xfrm_migrate *m, int num_migrate)
2647 {
2648         struct xfrm_migrate *mp;
2649         struct dst_entry *dst;
2650         int i, j, n = 0;
2651
2652         write_lock_bh(&pol->lock);
2653         if (unlikely(pol->walk.dead)) {
2654                 /* target policy has been deleted */
2655                 write_unlock_bh(&pol->lock);
2656                 return -ENOENT;
2657         }
2658
2659         for (i = 0; i < pol->xfrm_nr; i++) {
2660                 for (j = 0, mp = m; j < num_migrate; j++, mp++) {
2661                         if (!migrate_tmpl_match(mp, &pol->xfrm_vec[i]))
2662                                 continue;
2663                         n++;
2664                         if (pol->xfrm_vec[i].mode != XFRM_MODE_TUNNEL &&
2665                             pol->xfrm_vec[i].mode != XFRM_MODE_BEET)
2666                                 continue;
2667                         /* update endpoints */
2668                         memcpy(&pol->xfrm_vec[i].id.daddr, &mp->new_daddr,
2669                                sizeof(pol->xfrm_vec[i].id.daddr));
2670                         memcpy(&pol->xfrm_vec[i].saddr, &mp->new_saddr,
2671                                sizeof(pol->xfrm_vec[i].saddr));
2672                         pol->xfrm_vec[i].encap_family = mp->new_family;
2673                         /* flush bundles */
2674                         while ((dst = pol->bundles) != NULL) {
2675                                 pol->bundles = dst->next;
2676                                 dst_free(dst);
2677                         }
2678                 }
2679         }
2680
2681         write_unlock_bh(&pol->lock);
2682
2683         if (!n)
2684                 return -ENODATA;
2685
2686         return 0;
2687 }
2688
2689 static int xfrm_migrate_check(struct xfrm_migrate *m, int num_migrate)
2690 {
2691         int i, j;
2692
2693         if (num_migrate < 1 || num_migrate > XFRM_MAX_DEPTH)
2694                 return -EINVAL;
2695
2696         for (i = 0; i < num_migrate; i++) {
2697                 if ((xfrm_addr_cmp(&m[i].old_daddr, &m[i].new_daddr,
2698                                    m[i].old_family) == 0) &&
2699                     (xfrm_addr_cmp(&m[i].old_saddr, &m[i].new_saddr,
2700                                    m[i].old_family) == 0))
2701                         return -EINVAL;
2702                 if (xfrm_addr_any(&m[i].new_daddr, m[i].new_family) ||
2703                     xfrm_addr_any(&m[i].new_saddr, m[i].new_family))
2704                         return -EINVAL;
2705
2706                 /* check if there is any duplicated entry */
2707                 for (j = i + 1; j < num_migrate; j++) {
2708                         if (!memcmp(&m[i].old_daddr, &m[j].old_daddr,
2709                                     sizeof(m[i].old_daddr)) &&
2710                             !memcmp(&m[i].old_saddr, &m[j].old_saddr,
2711                                     sizeof(m[i].old_saddr)) &&
2712                             m[i].proto == m[j].proto &&
2713                             m[i].mode == m[j].mode &&
2714                             m[i].reqid == m[j].reqid &&
2715                             m[i].old_family == m[j].old_family)
2716                                 return -EINVAL;
2717                 }
2718         }
2719
2720         return 0;
2721 }
2722
2723 int xfrm_migrate(struct xfrm_selector *sel, u8 dir, u8 type,
2724                  struct xfrm_migrate *m, int num_migrate,
2725                  struct xfrm_kmaddress *k)
2726 {
2727         int i, err, nx_cur = 0, nx_new = 0;
2728         struct xfrm_policy *pol = NULL;
2729         struct xfrm_state *x, *xc;
2730         struct xfrm_state *x_cur[XFRM_MAX_DEPTH];
2731         struct xfrm_state *x_new[XFRM_MAX_DEPTH];
2732         struct xfrm_migrate *mp;
2733
2734         if ((err = xfrm_migrate_check(m, num_migrate)) < 0)
2735                 goto out;
2736
2737         /* Stage 1 - find policy */
2738         if ((pol = xfrm_migrate_policy_find(sel, dir, type)) == NULL) {
2739                 err = -ENOENT;
2740                 goto out;
2741         }
2742
2743         /* Stage 2 - find and update state(s) */
2744         for (i = 0, mp = m; i < num_migrate; i++, mp++) {
2745                 if ((x = xfrm_migrate_state_find(mp))) {
2746                         x_cur[nx_cur] = x;
2747                         nx_cur++;
2748                         if ((xc = xfrm_state_migrate(x, mp))) {
2749                                 x_new[nx_new] = xc;
2750                                 nx_new++;
2751                         } else {
2752                                 err = -ENODATA;
2753                                 goto restore_state;
2754                         }
2755                 }
2756         }
2757
2758         /* Stage 3 - update policy */
2759         if ((err = xfrm_policy_migrate(pol, m, num_migrate)) < 0)
2760                 goto restore_state;
2761
2762         /* Stage 4 - delete old state(s) */
2763         if (nx_cur) {
2764                 xfrm_states_put(x_cur, nx_cur);
2765                 xfrm_states_delete(x_cur, nx_cur);
2766         }
2767
2768         /* Stage 5 - announce */
2769         km_migrate(sel, dir, type, m, num_migrate, k);
2770
2771         xfrm_pol_put(pol);
2772
2773         return 0;
2774 out:
2775         return err;
2776
2777 restore_state:
2778         if (pol)
2779                 xfrm_pol_put(pol);
2780         if (nx_cur)
2781                 xfrm_states_put(x_cur, nx_cur);
2782         if (nx_new)
2783                 xfrm_states_delete(x_new, nx_new);
2784
2785         return err;
2786 }
2787 EXPORT_SYMBOL(xfrm_migrate);
2788 #endif