llvm.org GIT mirror llvm / release_60 lib / Target / AMDGPU / AMDGPUOpenCLEnqueuedBlockLowering.cpp

Tree @release_60 (Download .tar.gz)

AMDGPUOpenCLEnqueuedBlockLowering.cpp @release_60raw · history · blame

//===- AMDGPUOpenCLEnqueuedBlockLowering.cpp - Lower enqueued block -------===//
//                     The LLVM Compiler Infrastructure
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
// \file
// \brief This post-linking pass replaces the function pointer of enqueued
// block kernel with a global variable (runtime handle) and adds
// "runtime-handle" attribute to the enqueued block kernel.
// In LLVM CodeGen the runtime-handle metadata will be translated to
// RuntimeHandle metadata in code object. Runtime allocates a global buffer
// for each kernel with RuntimeHandel metadata and saves the kernel address
// required for the AQL packet into the buffer. __enqueue_kernel function
// in device library knows that the invoke function pointer in the block
// literal is actually runtime handle and loads the kernel address from it
// and put it into AQL packet for dispatching.
// This cannot be done in FE since FE cannot create a unique global variable
// with external linkage across LLVM modules. The global variable with internal
// linkage does not work since optimization passes will try to replace loads
// of the global variable with its initialization value.
// It also identifies the kernels directly or indirectly enqueues kernels
// and adds "calls-enqueue-kernel" function attribute to them, which will
// be used to determine whether to emit runtime metadata for the kernel
// enqueue related hidden kernel arguments.

#include "AMDGPU.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/User.h"
#include "llvm/Pass.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/raw_ostream.h"

#define DEBUG_TYPE "amdgpu-lower-enqueued-block"

using namespace llvm;

namespace {

/// \brief Lower enqueued blocks.
class AMDGPUOpenCLEnqueuedBlockLowering : public ModulePass {
  static char ID;

  explicit AMDGPUOpenCLEnqueuedBlockLowering() : ModulePass(ID) {}

  bool runOnModule(Module &M) override;

} // end anonymous namespace

char AMDGPUOpenCLEnqueuedBlockLowering::ID = 0;

char &llvm::AMDGPUOpenCLEnqueuedBlockLoweringID =

                "Lower OpenCL enqueued blocks", false, false)

ModulePass* llvm::createAMDGPUOpenCLEnqueuedBlockLoweringPass() {
  return new AMDGPUOpenCLEnqueuedBlockLowering();

/// Collect direct or indrect callers of \p F and save them
/// to \p Callers.
static void collectCallers(Function *F, DenseSet<Function *> &Callers) {
  for (auto U : F->users()) {
    if (auto *CI = dyn_cast<CallInst>(&*U)) {
      auto *Caller = CI->getParent()->getParent();
      if (Callers.count(Caller))
      collectCallers(Caller, Callers);

bool AMDGPUOpenCLEnqueuedBlockLowering::runOnModule(Module &M) {
  DenseSet<Function *> Callers;
  auto &C = M.getContext();
  bool Changed = false;
  for (auto &F : M.functions()) {
    if (F.hasFnAttribute("enqueued-block")) {
      if (!F.hasOneUse() || !F.user_begin()->hasOneUse() ||
          !isa<ConstantExpr>(*F.user_begin()) ||
          !isa<ConstantExpr>(*F.user_begin()->user_begin())) {
      auto *BitCast = cast<ConstantExpr>(*F.user_begin());
      auto *AddrCast = cast<ConstantExpr>(*BitCast->user_begin());
      auto RuntimeHandle = (F.getName() + "_runtime_handle").str();
      auto *GV = new GlobalVariable(
          M, Type::getInt8Ty(C)->getPointerTo(AMDGPUAS::GLOBAL_ADDRESS),
          /*IsConstant=*/true, GlobalValue::ExternalLinkage,
          /*Initializer=*/nullptr, RuntimeHandle, /*InsertBefore=*/nullptr,
          GlobalValue::NotThreadLocal, AMDGPUAS::GLOBAL_ADDRESS,
      DEBUG(dbgs() << "runtime handle created: " << *GV << '\n');
      auto *NewPtr = ConstantExpr::getPointerCast(GV, AddrCast->getType());
      F.addFnAttr("runtime-handle", RuntimeHandle);

      // Collect direct or indirect callers of enqueue_kernel.
      for (auto U : NewPtr->users()) {
        if (auto *I = dyn_cast<Instruction>(&*U)) {
          auto *F = I->getParent()->getParent();
          collectCallers(F, Callers);
      Changed = true;

  for (auto F : Callers) {
    if (F->getCallingConv() != CallingConv::AMDGPU_KERNEL)
  return Changed;