drivers/misc/sgi-xp/xpc_main.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (c) 2004-2008 Silicon Graphics, Inc.  All Rights Reserved.
   7  */
   8
   9 /*
  10  * Cross Partition Communication (XPC) support - standard version.
  11  *
  12  *      XPC provides a message passing capability that crosses partition
  13  *      boundaries. This module is made up of two parts:
  14  *
  15  *          partition   This part detects the presence/absence of other
  16  *                      partitions. It provides a heartbeat and monitors
  17  *                      the heartbeats of other partitions.
  18  *
  19  *          channel     This part manages the channels and sends/receives
  20  *                      messages across them to/from other partitions.
  21  *
  22  *      There are a couple of additional functions residing in XP, which
  23  *      provide an interface to XPC for its users.
  24  *
  25  *
  26  *      Caveats:
  27  *
  28  *        . Currently on sn2, we have no way to determine which nasid an IRQ
  29  *          came from. Thus, xpc_send_IRQ_sn2() does a remote AMO write
  30  *          followed by an IPI. The AMO indicates where data is to be pulled
  31  *          from, so after the IPI arrives, the remote partition checks the AMO
  32  *          word. The IPI can actually arrive before the AMO however, so other
  33  *          code must periodically check for this case. Also, remote AMO
  34  *          operations do not reliably time out. Thus we do a remote PIO read
  35  *          solely to know whether the remote partition is down and whether we
  36  *          should stop sending IPIs to it. This remote PIO read operation is
  37  *          set up in a special nofault region so SAL knows to ignore (and
  38  *          cleanup) any errors due to the remote AMO write, PIO read, and/or
  39  *          PIO write operations.
  40  *
  41  *          If/when new hardware solves this IPI problem, we should abandon
  42  *          the current approach.
  43  *
  44  */
  45
  46 #include <linux/kernel.h>
  47 #include <linux/module.h>
  48 #include <linux/init.h>
  49 #include <linux/cache.h>
  50 #include <linux/interrupt.h>
  51 #include <linux/delay.h>
  52 #include <linux/reboot.h>
  53 #include <linux/completion.h>
  54 #include <linux/kdebug.h>
  55 #include <linux/kthread.h>
  56 #include <linux/uaccess.h>
  57 #include <asm/sn/intr.h>
  58 #include <asm/sn/sn_sal.h>
  59 #include "xpc.h"
  60
  61 /* define two XPC debug device structures to be used with dev_dbg() et al */
  62
  63 struct device_driver xpc_dbg_name = {
  64         .name = "xpc"
  65 };
  66
  67 struct device xpc_part_dbg_subname = {
  68         .bus_id = {0},          /* set to "part" at xpc_init() time */
  69         .driver = &xpc_dbg_name
  70 };
  71
  72 struct device xpc_chan_dbg_subname = {
  73         .bus_id = {0},          /* set to "chan" at xpc_init() time */
  74         .driver = &xpc_dbg_name
  75 };
  76
  77 struct device *xpc_part = &xpc_part_dbg_subname;
  78 struct device *xpc_chan = &xpc_chan_dbg_subname;
  79
  80 static int xpc_kdebug_ignore;
  81
  82 /* systune related variables for /proc/sys directories */
  83
  84 static int xpc_hb_interval = XPC_HB_DEFAULT_INTERVAL;
  85 static int xpc_hb_min_interval = 1;
  86 static int xpc_hb_max_interval = 10;
  87
  88 static int xpc_hb_check_interval = XPC_HB_CHECK_DEFAULT_INTERVAL;
  89 static int xpc_hb_check_min_interval = 10;
  90 static int xpc_hb_check_max_interval = 120;
  91
  92 int xpc_disengage_timelimit = XPC_DISENGAGE_DEFAULT_TIMELIMIT;
  93 static int xpc_disengage_min_timelimit; /* = 0 */
  94 static int xpc_disengage_max_timelimit = 120;
  95
  96 static ctl_table xpc_sys_xpc_hb_dir[] = {
  97         {
  98          .ctl_name = CTL_UNNUMBERED,
  99          .procname = "hb_interval",
 100          .data = &xpc_hb_interval,
 101          .maxlen = sizeof(int),
 102          .mode = 0644,
 103          .proc_handler = &proc_dointvec_minmax,
 104          .strategy = &sysctl_intvec,
 105          .extra1 = &xpc_hb_min_interval,
 106          .extra2 = &xpc_hb_max_interval},
 107         {
 108          .ctl_name = CTL_UNNUMBERED,
 109          .procname = "hb_check_interval",
 110          .data = &xpc_hb_check_interval,
 111          .maxlen = sizeof(int),
 112          .mode = 0644,
 113          .proc_handler = &proc_dointvec_minmax,
 114          .strategy = &sysctl_intvec,
 115          .extra1 = &xpc_hb_check_min_interval,
 116          .extra2 = &xpc_hb_check_max_interval},
 117         {}
 118 };
 119 static ctl_table xpc_sys_xpc_dir[] = {
 120         {
 121          .ctl_name = CTL_UNNUMBERED,
 122          .procname = "hb",
 123          .mode = 0555,
 124          .child = xpc_sys_xpc_hb_dir},
 125         {
 126          .ctl_name = CTL_UNNUMBERED,
 127          .procname = "disengage_timelimit",
 128          .data = &xpc_disengage_timelimit,
 129          .maxlen = sizeof(int),
 130          .mode = 0644,
 131          .proc_handler = &proc_dointvec_minmax,
 132          .strategy = &sysctl_intvec,
 133          .extra1 = &xpc_disengage_min_timelimit,
 134          .extra2 = &xpc_disengage_max_timelimit},
 135         {}
 136 };
 137 static ctl_table xpc_sys_dir[] = {
 138         {
 139          .ctl_name = CTL_UNNUMBERED,
 140          .procname = "xpc",
 141          .mode = 0555,
 142          .child = xpc_sys_xpc_dir},
 143         {}
 144 };
 145 static struct ctl_table_header *xpc_sysctl;
 146
 147 /* non-zero if any remote partition disengage was timed out */
 148 int xpc_disengage_timedout;
 149
 150 /* #of activate IRQs received */
 151 atomic_t xpc_activate_IRQ_rcvd = ATOMIC_INIT(0);
 152
 153 /* IRQ handler notifies this wait queue on receipt of an IRQ */
 154 DECLARE_WAIT_QUEUE_HEAD(xpc_activate_IRQ_wq);
 155
 156 static unsigned long xpc_hb_check_timeout;
 157 static struct timer_list xpc_hb_timer;
 158 void *xpc_heartbeating_to_mask;
 159
 160 /* notification that the xpc_hb_checker thread has exited */
 161 static DECLARE_COMPLETION(xpc_hb_checker_exited);
 162
 163 /* notification that the xpc_discovery thread has exited */
 164 static DECLARE_COMPLETION(xpc_discovery_exited);
 165
 166 static void xpc_kthread_waitmsgs(struct xpc_partition *, struct xpc_channel *);
 167
 168 static int xpc_system_reboot(struct notifier_block *, unsigned long, void *);
 169 static struct notifier_block xpc_reboot_notifier = {
 170         .notifier_call = xpc_system_reboot,
 171 };
 172
 173 static int xpc_system_die(struct notifier_block *, unsigned long, void *);
 174 static struct notifier_block xpc_die_notifier = {
 175         .notifier_call = xpc_system_die,
 176 };
 177
 178 enum xp_retval (*xpc_rsvd_page_init) (struct xpc_rsvd_page *rp);
 179 void (*xpc_heartbeat_init) (void);
 180 void (*xpc_heartbeat_exit) (void);
 181 void (*xpc_increment_heartbeat) (void);
 182 void (*xpc_offline_heartbeat) (void);
 183 void (*xpc_online_heartbeat) (void);
 184 void (*xpc_check_remote_hb) (void);
 185
 186 enum xp_retval (*xpc_make_first_contact) (struct xpc_partition *part);
 187 void (*xpc_notify_senders_of_disconnect) (struct xpc_channel *ch);
 188 u64 (*xpc_get_chctl_all_flags) (struct xpc_partition *part);
 189 void (*xpc_process_msg_chctl_flags) (struct xpc_partition *part, int ch_number);
 190 int (*xpc_n_of_deliverable_msgs) (struct xpc_channel *ch);
 191 struct xpc_msg *(*xpc_get_deliverable_msg) (struct xpc_channel *ch);
 192
 193 void (*xpc_request_partition_activation) (struct xpc_rsvd_page *remote_rp,
 194                                           u64 remote_rp_pa, int nasid);
 195 void (*xpc_request_partition_reactivation) (struct xpc_partition *part);
 196 void (*xpc_request_partition_deactivation) (struct xpc_partition *part);
 197 void (*xpc_cancel_partition_deactivation_request) (struct xpc_partition *part);
 198
 199 void (*xpc_process_activate_IRQ_rcvd) (int n_IRQs_expected);
 200 enum xp_retval (*xpc_setup_infrastructure) (struct xpc_partition *part);
 201 void (*xpc_teardown_infrastructure) (struct xpc_partition *part);
 202
 203 void (*xpc_indicate_partition_engaged) (struct xpc_partition *part);
 204 int (*xpc_partition_engaged) (short partid);
 205 int (*xpc_any_partition_engaged) (void);
 206 void (*xpc_indicate_partition_disengaged) (struct xpc_partition *part);
 207 void (*xpc_assume_partition_disengaged) (short partid);
 208
 209 void (*xpc_send_chctl_closerequest) (struct xpc_channel *ch,
 210                                      unsigned long *irq_flags);
 211 void (*xpc_send_chctl_closereply) (struct xpc_channel *ch,
 212                                    unsigned long *irq_flags);
 213 void (*xpc_send_chctl_openrequest) (struct xpc_channel *ch,
 214                                     unsigned long *irq_flags);
 215 void (*xpc_send_chctl_openreply) (struct xpc_channel *ch,
 216                                   unsigned long *irq_flags);
 217
 218 enum xp_retval (*xpc_send_msg) (struct xpc_channel *ch, u32 flags,
 219                                 void *payload, u16 payload_size, u8 notify_type,
 220                                 xpc_notify_func func, void *key);
 221 void (*xpc_received_msg) (struct xpc_channel *ch, struct xpc_msg *msg);
 222
 223 /*
 224  * Timer function to enforce the timelimit on the partition disengage.
 225  */
 226 static void
 227 xpc_timeout_partition_disengage(unsigned long data)
 228 {
 229         struct xpc_partition *part = (struct xpc_partition *)data;
 230
 231         DBUG_ON(time_is_after_jiffies(part->disengage_timeout));
 232
 233         (void)xpc_partition_disengaged(part);
 234
 235         DBUG_ON(part->disengage_timeout != 0);
 236         DBUG_ON(xpc_partition_engaged(XPC_PARTID(part)));
 237 }
 238
 239 /*
 240  * Timer to produce the heartbeat.  The timer structures function is
 241  * already set when this is initially called.  A tunable is used to
 242  * specify when the next timeout should occur.
 243  */
 244 static void
 245 xpc_hb_beater(unsigned long dummy)
 246 {
 247         xpc_increment_heartbeat();
 248
 249         if (time_is_before_eq_jiffies(xpc_hb_check_timeout))
 250                 wake_up_interruptible(&xpc_activate_IRQ_wq);
 251
 252         xpc_hb_timer.expires = jiffies + (xpc_hb_interval * HZ);
 253         add_timer(&xpc_hb_timer);
 254 }
 255
 256 static void
 257 xpc_start_hb_beater(void)
 258 {
 259         xpc_heartbeat_init();
 260         init_timer(&xpc_hb_timer);
 261         xpc_hb_timer.function = xpc_hb_beater;
 262         xpc_hb_beater(0);
 263 }
 264
 265 static void
 266 xpc_stop_hb_beater(void)
 267 {
 268         del_timer_sync(&xpc_hb_timer);
 269         xpc_heartbeat_exit();
 270 }
 271
 272 /*
 273  * This thread is responsible for nearly all of the partition
 274  * activation/deactivation.
 275  */
 276 static int
 277 xpc_hb_checker(void *ignore)
 278 {
 279         int last_IRQ_count = 0;
 280         int new_IRQ_count;
 281         int force_IRQ = 0;
 282
 283         /* this thread was marked active by xpc_hb_init() */
 284
 285         set_cpus_allowed_ptr(current, &cpumask_of_cpu(XPC_HB_CHECK_CPU));
 286
 287         /* set our heartbeating to other partitions into motion */
 288         xpc_hb_check_timeout = jiffies + (xpc_hb_check_interval * HZ);
 289         xpc_start_hb_beater();
 290
 291         while (!xpc_exiting) {
 292
 293                 dev_dbg(xpc_part, "woke up with %d ticks rem; %d IRQs have "
 294                         "been received\n",
 295                         (int)(xpc_hb_check_timeout - jiffies),
 296                         atomic_read(&xpc_activate_IRQ_rcvd) - last_IRQ_count);
 297
 298                 /* checking of remote heartbeats is skewed by IRQ handling */
 299                 if (time_is_before_eq_jiffies(xpc_hb_check_timeout)) {
 300                         dev_dbg(xpc_part, "checking remote heartbeats\n");
 301                         xpc_check_remote_hb();
 302
 303                         /*
 304                          * We need to periodically recheck to ensure no
 305                          * IRQ/AMO pairs have been missed.  That check
 306                          * must always reset xpc_hb_check_timeout.
 307                          */
 308                         force_IRQ = 1;
 309                 }
 310
 311                 /* check for outstanding IRQs */
 312                 new_IRQ_count = atomic_read(&xpc_activate_IRQ_rcvd);
 313                 if (last_IRQ_count < new_IRQ_count || force_IRQ != 0) {
 314                         force_IRQ = 0;
 315
 316                         dev_dbg(xpc_part, "found an IRQ to process; will be "
 317                                 "resetting xpc_hb_check_timeout\n");
 318
 319                         xpc_process_activate_IRQ_rcvd(new_IRQ_count -
 320                                                       last_IRQ_count);
 321                         last_IRQ_count = new_IRQ_count;
 322
 323                         xpc_hb_check_timeout = jiffies +
 324                             (xpc_hb_check_interval * HZ);
 325                 }
 326
 327                 /* wait for IRQ or timeout */
 328                 (void)wait_event_interruptible(xpc_activate_IRQ_wq,
 329                                                (last_IRQ_count < atomic_read(
 330                                                 &xpc_activate_IRQ_rcvd)
 331                                                 || time_is_before_eq_jiffies(
 332                                                 xpc_hb_check_timeout) ||
 333                                                 xpc_exiting));
 334         }
 335
 336         xpc_stop_hb_beater();
 337
 338         dev_dbg(xpc_part, "heartbeat checker is exiting\n");
 339
 340         /* mark this thread as having exited */
 341         complete(&xpc_hb_checker_exited);
 342         return 0;
 343 }
 344
 345 /*
 346  * This thread will attempt to discover other partitions to activate
 347  * based on info provided by SAL. This new thread is short lived and
 348  * will exit once discovery is complete.
 349  */
 350 static int
 351 xpc_initiate_discovery(void *ignore)
 352 {
 353         xpc_discovery();
 354
 355         dev_dbg(xpc_part, "discovery thread is exiting\n");
 356
 357         /* mark this thread as having exited */
 358         complete(&xpc_discovery_exited);
 359         return 0;
 360 }
 361
 362 /*
 363  * The first kthread assigned to a newly activated partition is the one
 364  * created by XPC HB with which it calls xpc_activating(). XPC hangs on to
 365  * that kthread until the partition is brought down, at which time that kthread
 366  * returns back to XPC HB. (The return of that kthread will signify to XPC HB
 367  * that XPC has dismantled all communication infrastructure for the associated
 368  * partition.) This kthread becomes the channel manager for that partition.
 369  *
 370  * Each active partition has a channel manager, who, besides connecting and
 371  * disconnecting channels, will ensure that each of the partition's connected
 372  * channels has the required number of assigned kthreads to get the work done.
 373  */
 374 static void
 375 xpc_channel_mgr(struct xpc_partition *part)
 376 {
 377         while (part->act_state != XPC_P_DEACTIVATING ||
 378                atomic_read(&part->nchannels_active) > 0 ||
 379                !xpc_partition_disengaged(part)) {
 380
 381                 xpc_process_sent_chctl_flags(part);
 382
 383                 /*
 384                  * Wait until we've been requested to activate kthreads or
 385                  * all of the channel's message queues have been torn down or
 386                  * a signal is pending.
 387                  *
 388                  * The channel_mgr_requests is set to 1 after being awakened,
 389                  * This is done to prevent the channel mgr from making one pass
 390                  * through the loop for each request, since he will
 391                  * be servicing all the requests in one pass. The reason it's
 392                  * set to 1 instead of 0 is so that other kthreads will know
 393                  * that the channel mgr is running and won't bother trying to
 394                  * wake him up.
 395                  */
 396                 atomic_dec(&part->channel_mgr_requests);
 397                 (void)wait_event_interruptible(part->channel_mgr_wq,
 398                                 (atomic_read(&part->channel_mgr_requests) > 0 ||
 399                                  part->chctl.all_flags != 0 ||
 400                                  (part->act_state == XPC_P_DEACTIVATING &&
 401                                  atomic_read(&part->nchannels_active) == 0 &&
 402                                  xpc_partition_disengaged(part))));
 403                 atomic_set(&part->channel_mgr_requests, 1);
 404         }
 405 }
 406
 407 /*
 408  * When XPC HB determines that a partition has come up, it will create a new
 409  * kthread and that kthread will call this function to attempt to set up the
 410  * basic infrastructure used for Cross Partition Communication with the newly
 411  * upped partition.
 412  *
 413  * The kthread that was created by XPC HB and which setup the XPC
 414  * infrastructure will remain assigned to the partition becoming the channel
 415  * manager for that partition until the partition is deactivating, at which
 416  * time the kthread will teardown the XPC infrastructure and then exit.
 417  */
 418 static int
 419 xpc_activating(void *__partid)
 420 {
 421         short partid = (u64)__partid;
 422         struct xpc_partition *part = &xpc_partitions[partid];
 423         unsigned long irq_flags;
 424
 425         DBUG_ON(partid < 0 || partid >= xp_max_npartitions);
 426
 427         spin_lock_irqsave(&part->act_lock, irq_flags);
 428
 429         if (part->act_state == XPC_P_DEACTIVATING) {
 430                 part->act_state = XPC_P_INACTIVE;
 431                 spin_unlock_irqrestore(&part->act_lock, irq_flags);
 432                 part->remote_rp_pa = 0;
 433                 return 0;
 434         }
 435
 436         /* indicate the thread is activating */
 437         DBUG_ON(part->act_state != XPC_P_ACTIVATION_REQ);
 438         part->act_state = XPC_P_ACTIVATING;
 439
 440         XPC_SET_REASON(part, 0, 0);
 441         spin_unlock_irqrestore(&part->act_lock, irq_flags);
 442
 443         dev_dbg(xpc_part, "activating partition %d\n", partid);
 444
 445         xpc_allow_hb(partid);
 446
 447         if (xpc_setup_infrastructure(part) == xpSuccess) {
 448                 (void)xpc_part_ref(part);       /* this will always succeed */
 449
 450                 if (xpc_make_first_contact(part) == xpSuccess) {
 451                         xpc_mark_partition_active(part);
 452                         xpc_channel_mgr(part);
 453                         /* won't return until partition is deactivating */
 454                 }
 455
 456                 xpc_part_deref(part);
 457                 xpc_teardown_infrastructure(part);
 458         }
 459
 460         xpc_disallow_hb(partid);
 461         xpc_mark_partition_inactive(part);
 462
 463         if (part->reason == xpReactivating) {
 464                 /* interrupting ourselves results in activating partition */
 465                 xpc_request_partition_reactivation(part);
 466         }
 467
 468         return 0;
 469 }
 470
 471 void
 472 xpc_activate_partition(struct xpc_partition *part)
 473 {
 474         short partid = XPC_PARTID(part);
 475         unsigned long irq_flags;
 476         struct task_struct *kthread;
 477
 478         spin_lock_irqsave(&part->act_lock, irq_flags);
 479
 480         DBUG_ON(part->act_state != XPC_P_INACTIVE);
 481
 482         part->act_state = XPC_P_ACTIVATION_REQ;
 483         XPC_SET_REASON(part, xpCloneKThread, __LINE__);
 484
 485         spin_unlock_irqrestore(&part->act_lock, irq_flags);
 486
 487         kthread = kthread_run(xpc_activating, (void *)((u64)partid), "xpc%02d",
 488                               partid);
 489         if (IS_ERR(kthread)) {
 490                 spin_lock_irqsave(&part->act_lock, irq_flags);
 491                 part->act_state = XPC_P_INACTIVE;
 492                 XPC_SET_REASON(part, xpCloneKThreadFailed, __LINE__);
 493                 spin_unlock_irqrestore(&part->act_lock, irq_flags);
 494         }
 495 }
 496
 497 void
 498 xpc_activate_kthreads(struct xpc_channel *ch, int needed)
 499 {
 500         int idle = atomic_read(&ch->kthreads_idle);
 501         int assigned = atomic_read(&ch->kthreads_assigned);
 502         int wakeup;
 503
 504         DBUG_ON(needed <= 0);
 505
 506         if (idle > 0) {
 507                 wakeup = (needed > idle) ? idle : needed;
 508                 needed -= wakeup;
 509
 510                 dev_dbg(xpc_chan, "wakeup %d idle kthreads, partid=%d, "
 511                         "channel=%d\n", wakeup, ch->partid, ch->number);
 512
 513                 /* only wakeup the requested number of kthreads */
 514                 wake_up_nr(&ch->idle_wq, wakeup);
 515         }
 516
 517         if (needed <= 0)
 518                 return;
 519
 520         if (needed + assigned > ch->kthreads_assigned_limit) {
 521                 needed = ch->kthreads_assigned_limit - assigned;
 522                 if (needed <= 0)
 523                         return;
 524         }
 525
 526         dev_dbg(xpc_chan, "create %d new kthreads, partid=%d, channel=%d\n",
 527                 needed, ch->partid, ch->number);
 528
 529         xpc_create_kthreads(ch, needed, 0);
 530 }
 531
 532 /*
 533  * This function is where XPC's kthreads wait for messages to deliver.
 534  */
 535 static void
 536 xpc_kthread_waitmsgs(struct xpc_partition *part, struct xpc_channel *ch)
 537 {
 538         do {
 539                 /* deliver messages to their intended recipients */
 540
 541                 while (xpc_n_of_deliverable_msgs(ch) > 0 &&
 542                        !(ch->flags & XPC_C_DISCONNECTING)) {
 543                         xpc_deliver_msg(ch);
 544                 }
 545
 546                 if (atomic_inc_return(&ch->kthreads_idle) >
 547                     ch->kthreads_idle_limit) {
 548                         /* too many idle kthreads on this channel */
 549                         atomic_dec(&ch->kthreads_idle);
 550                         break;
 551                 }
 552
 553                 dev_dbg(xpc_chan, "idle kthread calling "
 554                         "wait_event_interruptible_exclusive()\n");
 555
 556                 (void)wait_event_interruptible_exclusive(ch->idle_wq,
 557                                 (xpc_n_of_deliverable_msgs(ch) > 0 ||
 558                                  (ch->flags & XPC_C_DISCONNECTING)));
 559
 560                 atomic_dec(&ch->kthreads_idle);
 561
 562         } while (!(ch->flags & XPC_C_DISCONNECTING));
 563 }
 564
 565 static int
 566 xpc_kthread_start(void *args)
 567 {
 568         short partid = XPC_UNPACK_ARG1(args);
 569         u16 ch_number = XPC_UNPACK_ARG2(args);
 570         struct xpc_partition *part = &xpc_partitions[partid];
 571         struct xpc_channel *ch;
 572         int n_needed;
 573         unsigned long irq_flags;
 574
 575         dev_dbg(xpc_chan, "kthread starting, partid=%d, channel=%d\n",
 576                 partid, ch_number);
 577
 578         ch = &part->channels[ch_number];
 579
 580         if (!(ch->flags & XPC_C_DISCONNECTING)) {
 581
 582                 /* let registerer know that connection has been established */
 583
 584                 spin_lock_irqsave(&ch->lock, irq_flags);
 585                 if (!(ch->flags & XPC_C_CONNECTEDCALLOUT)) {
 586                         ch->flags |= XPC_C_CONNECTEDCALLOUT;
 587                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 588
 589                         xpc_connected_callout(ch);
 590
 591                         spin_lock_irqsave(&ch->lock, irq_flags);
 592                         ch->flags |= XPC_C_CONNECTEDCALLOUT_MADE;
 593                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 594
 595                         /*
 596                          * It is possible that while the callout was being
 597                          * made that the remote partition sent some messages.
 598                          * If that is the case, we may need to activate
 599                          * additional kthreads to help deliver them. We only
 600                          * need one less than total #of messages to deliver.
 601                          */
 602                         n_needed = xpc_n_of_deliverable_msgs(ch) - 1;
 603                         if (n_needed > 0 && !(ch->flags & XPC_C_DISCONNECTING))
 604                                 xpc_activate_kthreads(ch, n_needed);
 605
 606                 } else {
 607                         spin_unlock_irqrestore(&ch->lock, irq_flags);
 608                 }
 609
 610                 xpc_kthread_waitmsgs(part, ch);
 611         }
 612
 613         /* let registerer know that connection is disconnecting */
 614
 615         spin_lock_irqsave(&ch->lock, irq_flags);
 616         if ((ch->flags & XPC_C_CONNECTEDCALLOUT_MADE) &&
 617             !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) {
 618                 ch->flags |= XPC_C_DISCONNECTINGCALLOUT;
 619                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 620
 621                 xpc_disconnect_callout(ch, xpDisconnecting);
 622
 623                 spin_lock_irqsave(&ch->lock, irq_flags);
 624                 ch->flags |= XPC_C_DISCONNECTINGCALLOUT_MADE;
 625         }
 626         spin_unlock_irqrestore(&ch->lock, irq_flags);
 627
 628         if (atomic_dec_return(&ch->kthreads_assigned) == 0 &&
 629             atomic_dec_return(&part->nchannels_engaged) == 0) {
 630                 xpc_indicate_partition_disengaged(part);
 631         }
 632
 633         xpc_msgqueue_deref(ch);
 634
 635         dev_dbg(xpc_chan, "kthread exiting, partid=%d, channel=%d\n",
 636                 partid, ch_number);
 637
 638         xpc_part_deref(part);
 639         return 0;
 640 }
 641
 642 /*
 643  * For each partition that XPC has established communications with, there is
 644  * a minimum of one kernel thread assigned to perform any operation that
 645  * may potentially sleep or block (basically the callouts to the asynchronous
 646  * functions registered via xpc_connect()).
 647  *
 648  * Additional kthreads are created and destroyed by XPC as the workload
 649  * demands.
 650  *
 651  * A kthread is assigned to one of the active channels that exists for a given
 652  * partition.
 653  */
 654 void
 655 xpc_create_kthreads(struct xpc_channel *ch, int needed,
 656                     int ignore_disconnecting)
 657 {
 658         unsigned long irq_flags;
 659         u64 args = XPC_PACK_ARGS(ch->partid, ch->number);
 660         struct xpc_partition *part = &xpc_partitions[ch->partid];
 661         struct task_struct *kthread;
 662
 663         while (needed-- > 0) {
 664
 665                 /*
 666                  * The following is done on behalf of the newly created
 667                  * kthread. That kthread is responsible for doing the
 668                  * counterpart to the following before it exits.
 669                  */
 670                 if (ignore_disconnecting) {
 671                         if (!atomic_inc_not_zero(&ch->kthreads_assigned)) {
 672                                 /* kthreads assigned had gone to zero */
 673                                 BUG_ON(!(ch->flags &
 674                                          XPC_C_DISCONNECTINGCALLOUT_MADE));
 675                                 break;
 676                         }
 677
 678                 } else if (ch->flags & XPC_C_DISCONNECTING) {
 679                         break;
 680
 681                 } else if (atomic_inc_return(&ch->kthreads_assigned) == 1 &&
 682                            atomic_inc_return(&part->nchannels_engaged) == 1) {
 683                                 xpc_indicate_partition_engaged(part);
 684                 }
 685                 (void)xpc_part_ref(part);
 686                 xpc_msgqueue_ref(ch);
 687
 688                 kthread = kthread_run(xpc_kthread_start, (void *)args,
 689                                       "xpc%02dc%d", ch->partid, ch->number);
 690                 if (IS_ERR(kthread)) {
 691                         /* the fork failed */
 692
 693                         /*
 694                          * NOTE: if (ignore_disconnecting &&
 695                          * !(ch->flags & XPC_C_DISCONNECTINGCALLOUT)) is true,
 696                          * then we'll deadlock if all other kthreads assigned
 697                          * to this channel are blocked in the channel's
 698                          * registerer, because the only thing that will unblock
 699                          * them is the xpDisconnecting callout that this
 700                          * failed kthread_run() would have made.
 701                          */
 702
 703                         if (atomic_dec_return(&ch->kthreads_assigned) == 0 &&
 704                             atomic_dec_return(&part->nchannels_engaged) == 0) {
 705                                 xpc_indicate_partition_disengaged(part);
 706                         }
 707                         xpc_msgqueue_deref(ch);
 708                         xpc_part_deref(part);
 709
 710                         if (atomic_read(&ch->kthreads_assigned) <
 711                             ch->kthreads_idle_limit) {
 712                                 /*
 713                                  * Flag this as an error only if we have an
 714                                  * insufficient #of kthreads for the channel
 715                                  * to function.
 716                                  */
 717                                 spin_lock_irqsave(&ch->lock, irq_flags);
 718                                 XPC_DISCONNECT_CHANNEL(ch, xpLackOfResources,
 719                                                        &irq_flags);
 720                                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 721                         }
 722                         break;
 723                 }
 724         }
 725 }
 726
 727 void
 728 xpc_disconnect_wait(int ch_number)
 729 {
 730         unsigned long irq_flags;
 731         short partid;
 732         struct xpc_partition *part;
 733         struct xpc_channel *ch;
 734         int wakeup_channel_mgr;
 735
 736         /* now wait for all callouts to the caller's function to cease */
 737         for (partid = 0; partid < xp_max_npartitions; partid++) {
 738                 part = &xpc_partitions[partid];
 739
 740                 if (!xpc_part_ref(part))
 741                         continue;
 742
 743                 ch = &part->channels[ch_number];
 744
 745                 if (!(ch->flags & XPC_C_WDISCONNECT)) {
 746                         xpc_part_deref(part);
 747                         continue;
 748                 }
 749
 750                 wait_for_completion(&ch->wdisconnect_wait);
 751
 752                 spin_lock_irqsave(&ch->lock, irq_flags);
 753                 DBUG_ON(!(ch->flags & XPC_C_DISCONNECTED));
 754                 wakeup_channel_mgr = 0;
 755
 756                 if (ch->delayed_chctl_flags) {
 757                         if (part->act_state != XPC_P_DEACTIVATING) {
 758                                 spin_lock(&part->chctl_lock);
 759                                 part->chctl.flags[ch->number] |=
 760                                     ch->delayed_chctl_flags;
 761                                 spin_unlock(&part->chctl_lock);
 762                                 wakeup_channel_mgr = 1;
 763                         }
 764                         ch->delayed_chctl_flags = 0;
 765                 }
 766
 767                 ch->flags &= ~XPC_C_WDISCONNECT;
 768                 spin_unlock_irqrestore(&ch->lock, irq_flags);
 769
 770                 if (wakeup_channel_mgr)
 771                         xpc_wakeup_channel_mgr(part);
 772
 773                 xpc_part_deref(part);
 774         }
 775 }
 776
 777 static void
 778 xpc_do_exit(enum xp_retval reason)
 779 {
 780         short partid;
 781         int active_part_count, printed_waiting_msg = 0;
 782         struct xpc_partition *part;
 783         unsigned long printmsg_time, disengage_timeout = 0;
 784
 785         /* a 'rmmod XPC' and a 'reboot' cannot both end up here together */
 786         DBUG_ON(xpc_exiting == 1);
 787
 788         /*
 789          * Let the heartbeat checker thread and the discovery thread
 790          * (if one is running) know that they should exit. Also wake up
 791          * the heartbeat checker thread in case it's sleeping.
 792          */
 793         xpc_exiting = 1;
 794         wake_up_interruptible(&xpc_activate_IRQ_wq);
 795
 796         /* wait for the discovery thread to exit */
 797         wait_for_completion(&xpc_discovery_exited);
 798
 799         /* wait for the heartbeat checker thread to exit */
 800         wait_for_completion(&xpc_hb_checker_exited);
 801
 802         /* sleep for a 1/3 of a second or so */
 803         (void)msleep_interruptible(300);
 804
 805         /* wait for all partitions to become inactive */
 806
 807         printmsg_time = jiffies + (XPC_DEACTIVATE_PRINTMSG_INTERVAL * HZ);
 808         xpc_disengage_timedout = 0;
 809
 810         do {
 811                 active_part_count = 0;
 812
 813                 for (partid = 0; partid < xp_max_npartitions; partid++) {
 814                         part = &xpc_partitions[partid];
 815
 816                         if (xpc_partition_disengaged(part) &&
 817                             part->act_state == XPC_P_INACTIVE) {
 818                                 continue;
 819                         }
 820
 821                         active_part_count++;
 822
 823                         XPC_DEACTIVATE_PARTITION(part, reason);
 824
 825                         if (part->disengage_timeout > disengage_timeout)
 826                                 disengage_timeout = part->disengage_timeout;
 827                 }
 828
 829                 if (xpc_any_partition_engaged()) {
 830                         if (time_is_before_jiffies(printmsg_time)) {
 831                                 dev_info(xpc_part, "waiting for remote "
 832                                          "partitions to deactivate, timeout in "
 833                                          "%ld seconds\n", (disengage_timeout -
 834                                          jiffies) / HZ);
 835                                 printmsg_time = jiffies +
 836                                     (XPC_DEACTIVATE_PRINTMSG_INTERVAL * HZ);
 837                                 printed_waiting_msg = 1;
 838                         }
 839
 840                 } else if (active_part_count > 0) {
 841                         if (printed_waiting_msg) {
 842                                 dev_info(xpc_part, "waiting for local partition"
 843                                          " to deactivate\n");
 844                                 printed_waiting_msg = 0;
 845                         }
 846
 847                 } else {
 848                         if (!xpc_disengage_timedout) {
 849                                 dev_info(xpc_part, "all partitions have "
 850                                          "deactivated\n");
 851                         }
 852                         break;
 853                 }
 854
 855                 /* sleep for a 1/3 of a second or so */
 856                 (void)msleep_interruptible(300);
 857
 858         } while (1);
 859
 860         DBUG_ON(xpc_any_partition_engaged());
 861         DBUG_ON(xpc_any_hbs_allowed() != 0);
 862
 863         /* indicate to others that our reserved page is uninitialized */
 864         xpc_rsvd_page->stamp = 0;
 865
 866         if (reason == xpUnloading) {
 867                 (void)unregister_die_notifier(&xpc_die_notifier);
 868                 (void)unregister_reboot_notifier(&xpc_reboot_notifier);
 869         }
 870
 871         /* clear the interface to XPC's functions */
 872         xpc_clear_interface();
 873
 874         if (xpc_sysctl)
 875                 unregister_sysctl_table(xpc_sysctl);
 876
 877         kfree(xpc_partitions);
 878         kfree(xpc_remote_copy_buffer_base);
 879
 880         if (is_shub())
 881                 xpc_exit_sn2();
 882         else
 883                 xpc_exit_uv();
 884 }
 885
 886 /*
 887  * This function is called when the system is being rebooted.
 888  */
 889 static int
 890 xpc_system_reboot(struct notifier_block *nb, unsigned long event, void *unused)
 891 {
 892         enum xp_retval reason;
 893
 894         switch (event) {
 895         case SYS_RESTART:
 896                 reason = xpSystemReboot;
 897                 break;
 898         case SYS_HALT:
 899                 reason = xpSystemHalt;
 900                 break;
 901         case SYS_POWER_OFF:
 902                 reason = xpSystemPoweroff;
 903                 break;
 904         default:
 905                 reason = xpSystemGoingDown;
 906         }
 907
 908         xpc_do_exit(reason);
 909         return NOTIFY_DONE;
 910 }
 911
 912 /*
 913  * Notify other partitions to deactivate from us by first disengaging from all
 914  * references to our memory.
 915  */
 916 static void
 917 xpc_die_deactivate(void)
 918 {
 919         struct xpc_partition *part;
 920         short partid;
 921         int any_engaged;
 922         long time, printmsg_time, disengage_timeout;
 923
 924         /* keep xpc_hb_checker thread from doing anything (just in case) */
 925         xpc_exiting = 1;
 926
 927         xpc_disallow_all_hbs(); /*indicate we're deactivated */
 928
 929         for (partid = 0; partid < xp_max_npartitions; partid++) {
 930                 part = &xpc_partitions[partid];
 931
 932                 if (xpc_partition_engaged(partid) ||
 933                     part->act_state != XPC_P_INACTIVE) {
 934                         xpc_request_partition_deactivation(part);
 935                         xpc_indicate_partition_disengaged(part);
 936                 }
 937         }
 938
 939         time = rtc_time();
 940         printmsg_time = time +
 941             (XPC_DEACTIVATE_PRINTMSG_INTERVAL * sn_rtc_cycles_per_second);
 942         disengage_timeout = time +
 943             (xpc_disengage_timelimit * sn_rtc_cycles_per_second);
 944
 945         /*
 946          * Though we requested that all other partitions deactivate from us,
 947          * we only wait until they've all disengaged.
 948          */
 949
 950         while (1) {
 951                 any_engaged = xpc_any_partition_engaged();
 952                 if (!any_engaged) {
 953                         dev_info(xpc_part, "all partitions have deactivated\n");
 954                         break;
 955                 }
 956
 957                 time = rtc_time();
 958                 if (time >= disengage_timeout) {
 959                         for (partid = 0; partid < xp_max_npartitions;
 960                              partid++) {
 961                                 if (xpc_partition_engaged(partid)) {
 962                                         dev_info(xpc_part, "deactivate from "
 963                                                  "remote partition %d timed "
 964                                                  "out\n", partid);
 965                                 }
 966                         }
 967                         break;
 968                 }
 969
 970                 if (time >= printmsg_time) {
 971                         dev_info(xpc_part, "waiting for remote partitions to "
 972                                  "deactivate, timeout in %ld seconds\n",
 973                                  (disengage_timeout - time) /
 974                                  sn_rtc_cycles_per_second);
 975                         printmsg_time = time +
 976                             (XPC_DEACTIVATE_PRINTMSG_INTERVAL *
 977                              sn_rtc_cycles_per_second);
 978                 }
 979         }
 980 }
 981
 982 /*
 983  * This function is called when the system is being restarted or halted due
 984  * to some sort of system failure. If this is the case we need to notify the
 985  * other partitions to disengage from all references to our memory.
 986  * This function can also be called when our heartbeater could be offlined
 987  * for a time. In this case we need to notify other partitions to not worry
 988  * about the lack of a heartbeat.
 989  */
 990 static int
 991 xpc_system_die(struct notifier_block *nb, unsigned long event, void *unused)
 992 {
 993         switch (event) {
 994         case DIE_MACHINE_RESTART:
 995         case DIE_MACHINE_HALT:
 996                 xpc_die_deactivate();
 997                 break;
 998
 999         case DIE_KDEBUG_ENTER:
1000                 /* Should lack of heartbeat be ignored by other partitions? */
1001                 if (!xpc_kdebug_ignore)
1002                         break;
1003
1004                 /* fall through */
1005         case DIE_MCA_MONARCH_ENTER:
1006         case DIE_INIT_MONARCH_ENTER:
1007                 xpc_offline_heartbeat();
1008                 break;
1009
1010         case DIE_KDEBUG_LEAVE:
1011                 /* Is lack of heartbeat being ignored by other partitions? */
1012                 if (!xpc_kdebug_ignore)
1013                         break;
1014
1015                 /* fall through */
1016         case DIE_MCA_MONARCH_LEAVE:
1017         case DIE_INIT_MONARCH_LEAVE:
1018                 xpc_online_heartbeat();
1019                 break;
1020         }
1021
1022         return NOTIFY_DONE;
1023 }
1024
1025 int __init
1026 xpc_init(void)
1027 {
1028         int ret;
1029         short partid;
1030         struct xpc_partition *part;
1031         struct task_struct *kthread;
1032         size_t buf_size;
1033
1034         if (is_shub()) {
1035                 /*
1036                  * The ia64-sn2 architecture supports at most 64 partitions.
1037                  * And the inability to unregister remote AMOs restricts us
1038                  * further to only support exactly 64 partitions on this
1039                  * architecture, no less.
1040                  */
1041                 if (xp_max_npartitions != 64)
1042                         return -EINVAL;
1043
1044                 ret = xpc_init_sn2();
1045                 if (ret != 0)
1046                         return ret;
1047
1048         } else if (is_uv()) {
1049                 xpc_init_uv();
1050
1051         } else {
1052                 return -ENODEV;
1053         }
1054
1055         snprintf(xpc_part->bus_id, BUS_ID_SIZE, "part");
1056         snprintf(xpc_chan->bus_id, BUS_ID_SIZE, "chan");
1057
1058         buf_size = max(XPC_RP_VARS_SIZE,
1059                        XPC_RP_HEADER_SIZE + XP_NASID_MASK_BYTES);
1060         xpc_remote_copy_buffer = xpc_kmalloc_cacheline_aligned(buf_size,
1061                                                                GFP_KERNEL,
1062                                                   &xpc_remote_copy_buffer_base);
1063         if (xpc_remote_copy_buffer == NULL) {
1064                 dev_err(xpc_part, "can't get memory for remote copy buffer\n");
1065                 ret = -ENOMEM;
1066                 goto out_1;
1067         }
1068
1069         xpc_partitions = kzalloc(sizeof(struct xpc_partition) *
1070                                  xp_max_npartitions, GFP_KERNEL);
1071         if (xpc_partitions == NULL) {
1072                 dev_err(xpc_part, "can't get memory for partition structure\n");
1073                 ret = -ENOMEM;
1074                 goto out_2;
1075         }
1076
1077         /*
1078          * The first few fields of each entry of xpc_partitions[] need to
1079          * be initialized now so that calls to xpc_connect() and
1080          * xpc_disconnect() can be made prior to the activation of any remote
1081          * partition. NOTE THAT NONE OF THE OTHER FIELDS BELONGING TO THESE
1082          * ENTRIES ARE MEANINGFUL UNTIL AFTER AN ENTRY'S CORRESPONDING
1083          * PARTITION HAS BEEN ACTIVATED.
1084          */
1085         for (partid = 0; partid < xp_max_npartitions; partid++) {
1086                 part = &xpc_partitions[partid];
1087
1088                 DBUG_ON((u64)part != L1_CACHE_ALIGN((u64)part));
1089
1090                 part->activate_IRQ_rcvd = 0;
1091                 spin_lock_init(&part->act_lock);
1092                 part->act_state = XPC_P_INACTIVE;
1093                 XPC_SET_REASON(part, 0, 0);
1094
1095                 init_timer(&part->disengage_timer);
1096                 part->disengage_timer.function =
1097                     xpc_timeout_partition_disengage;
1098                 part->disengage_timer.data = (unsigned long)part;
1099
1100                 part->setup_state = XPC_P_UNSET;
1101                 init_waitqueue_head(&part->teardown_wq);
1102                 atomic_set(&part->references, 0);
1103         }
1104
1105         xpc_sysctl = register_sysctl_table(xpc_sys_dir);
1106
1107         /*
1108          * Fill the partition reserved page with the information needed by
1109          * other partitions to discover we are alive and establish initial
1110          * communications.
1111          */
1112         xpc_rsvd_page = xpc_setup_rsvd_page();
1113         if (xpc_rsvd_page == NULL) {
1114                 dev_err(xpc_part, "can't setup our reserved page\n");
1115                 ret = -EBUSY;
1116                 goto out_3;
1117         }
1118
1119         /* add ourselves to the reboot_notifier_list */
1120         ret = register_reboot_notifier(&xpc_reboot_notifier);
1121         if (ret != 0)
1122                 dev_warn(xpc_part, "can't register reboot notifier\n");
1123
1124         /* add ourselves to the die_notifier list */
1125         ret = register_die_notifier(&xpc_die_notifier);
1126         if (ret != 0)
1127                 dev_warn(xpc_part, "can't register die notifier\n");
1128
1129         /*
1130          * The real work-horse behind xpc.  This processes incoming
1131          * interrupts and monitors remote heartbeats.
1132          */
1133         kthread = kthread_run(xpc_hb_checker, NULL, XPC_HB_CHECK_THREAD_NAME);
1134         if (IS_ERR(kthread)) {
1135                 dev_err(xpc_part, "failed while forking hb check thread\n");
1136                 ret = -EBUSY;
1137                 goto out_4;
1138         }
1139
1140         /*
1141          * Startup a thread that will attempt to discover other partitions to
1142          * activate based on info provided by SAL. This new thread is short
1143          * lived and will exit once discovery is complete.
1144          */
1145         kthread = kthread_run(xpc_initiate_discovery, NULL,
1146                               XPC_DISCOVERY_THREAD_NAME);
1147         if (IS_ERR(kthread)) {
1148                 dev_err(xpc_part, "failed while forking discovery thread\n");
1149
1150                 /* mark this new thread as a non-starter */
1151                 complete(&xpc_discovery_exited);
1152
1153                 xpc_do_exit(xpUnloading);
1154                 return -EBUSY;
1155         }
1156
1157         /* set the interface to point at XPC's functions */
1158         xpc_set_interface(xpc_initiate_connect, xpc_initiate_disconnect,
1159                           xpc_initiate_send, xpc_initiate_send_notify,
1160                           xpc_initiate_received, xpc_initiate_partid_to_nasids);
1161
1162         return 0;
1163
1164         /* initialization was not successful */
1165 out_4:
1166         /* indicate to others that our reserved page is uninitialized */
1167         xpc_rsvd_page->stamp = 0;
1168
1169         (void)unregister_die_notifier(&xpc_die_notifier);
1170         (void)unregister_reboot_notifier(&xpc_reboot_notifier);
1171 out_3:
1172         if (xpc_sysctl)
1173                 unregister_sysctl_table(xpc_sysctl);
1174         kfree(xpc_partitions);
1175 out_2:
1176         kfree(xpc_remote_copy_buffer_base);
1177 out_1:
1178         if (is_shub())
1179                 xpc_exit_sn2();
1180         else
1181                 xpc_exit_uv();
1182         return ret;
1183 }
1184
1185 module_init(xpc_init);
1186
1187 void __exit
1188 xpc_exit(void)
1189 {
1190         xpc_do_exit(xpUnloading);
1191 }
1192
1193 module_exit(xpc_exit);
1194
1195 MODULE_AUTHOR("Silicon Graphics, Inc.");
1196 MODULE_DESCRIPTION("Cross Partition Communication (XPC) support");
1197 MODULE_LICENSE("GPL");
1198
1199 module_param(xpc_hb_interval, int, 0);
1200 MODULE_PARM_DESC(xpc_hb_interval, "Number of seconds between "
1201                  "heartbeat increments.");
1202
1203 module_param(xpc_hb_check_interval, int, 0);
1204 MODULE_PARM_DESC(xpc_hb_check_interval, "Number of seconds between "
1205                  "heartbeat checks.");
1206
1207 module_param(xpc_disengage_timelimit, int, 0);
1208 MODULE_PARM_DESC(xpc_disengage_timelimit, "Number of seconds to wait "
1209                  "for disengage to complete.");
1210
1211 module_param(xpc_kdebug_ignore, int, 0);
1212 MODULE_PARM_DESC(xpc_kdebug_ignore, "Should lack of heartbeat be ignored by "
1213                  "other partitions when dropping into kdebug.");